Change Log

  ==========

  Supported build systems

  =======================

    ``{'platform': ['f28', 'f29']}``.

  - ``require_overrides`` - the requires to override the modulemd with. The overrides must be to

    existing requires on the modulemd. The expected format is ``{'platform': ['f28', 'f29']}``.

  - ``yaml`` - a string of the input file when submitting a YAML file directly in a

    ``multipart/form-data`` request. The MBS setting ``YAML_SUBMIT_ALLOWED`` must be set to ``True``

    for this to be allowed.

  - ``rebuild_strategy`` - a string of the desired rebuild strategy (affects what components get

    rebuilt). For the available options, please look at the "Rebuild Strategies" section below.

  - ``owner`` - the username of the owner or person who submitted the module build.

  - ``scmurl`` - the source control URL used to build the module.

  - ``state`` - the numerical state of the module build.

  - ``state_reason`` - the reason why the module build is in this state. This is useful

    when the build fails.

  - ``stream`` - the module's stream.

    TestConfiguration is used, otherwise...

  - if ``MODULE_BUILD_SERVICE_DEVELOPER_ENV`` is set to some reasonable

    value, DevConfiguration is forced and ``config.py`` is used directly from the

  Setting Up Kerberos + LDAP Authentication

  Development

  ===========

  License

  =======

  Parts of MBS are licensed under 3-clause BSD license from:

  https://github.com/projectatomic/atomic-reactor/blob/master/LICENSE

      ARCHES = ['i686', 'armv7hl', 'x86_64']

      ALLOW_ARCH_OVERRIDE = False

      KOJI_REPOSITORY_URL = 'https://kojipkgs.fedoraproject.org/repos'

      KOJI_ENABLE_CONTENT_GENERATOR = True

      CHECK_FOR_EOL = False

      PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1'

      MODULES_DEFAULT_REPOSITORY = 'https://src.fedoraproject.org/modules/'

      MODULES_ALLOW_REPOSITORY = False

      ALLOWED_GROUPS = set([

          'packager',

  There is the MBS frontend, which provides a REST API (See ``views.py``). A user sends a POST request

  with JSON describing the module to build. There is mainly the URL to the git repository (called

  This JSON data is handled by ``views.SCMHandler``, which validates the JSON and calls

  ``utils.submit.submit_module_build_from_scm(...)`` method. This goes down to

  ``submit_module_build(...)``.

  Module Stream Expansion (MSE)

  -----------------------------

  This method imports the module build into Koji using the ``KojiContentGenerator`` class.

  The module build in Koji points to the Koji tag with the module's components and also contains the

  final modulemd files generated for earch architecture the module is built for.

                  raise RuntimeError("Buildroot is not prep-ed")

              self._koji_whitelist_packages([artifact_name])

                  # treat source as an srpm and upload it

                  serverdir = _unique_path('cli-build')

                  callback = None

              return list(nvrs)

      def finalize(self):

              cg = KojiContentGenerator(self.module, self.config)

              cg.koji_import()

              if conf.koji_cg_devel_module:

          tags = []

          koji_tags = session.listTags(rpm_md["build_id"])

          for t in koji_tags:

                  tags.append(t["name"])

          return tags

              'desc': 'Target to build "module-build-macros" RPM in.'},

          'koji_tag_prefixes': {

              'type': list,

              'desc': 'List of allowed koji tag prefixes.'},

          'koji_target_delete_time': {

              'type': int,

  @manager.option('--stream', action='store', dest="stream")

  @manager.option('--file', action='store', dest="yaml_file")

  @manager.option('--skiptests', action='store_true', dest="skiptests")

  @manager.option('-l', '--add-local-build', action='append', default=None, dest='local_build_nsvs')

  @manager.option('-s', '--set-stream', action='append', default=[], dest='default_streams')

      """ Performs local module build using Mock

      """

      if 'SERVER_NAME' not in app.config or not app.config['SERVER_NAME']:

          for ns in default_streams:

              name, stream = ns.split(":")

              optional_params["default_streams"][name] = stream

          username = getpass.getuser()

          if not yaml_file or not yaml_file.endswith(".yaml"):

      # Koji tag to which tag the Content Generator Koji build.

      cg_build_koji_tag = db.Column(db.String)  # This gets set after wait

      scmurl = db.Column(db.String)

      owner = db.Column(db.String, nullable=False)

      time_submitted = db.Column(db.DateTime, nullable=False)

      time_modified = db.Column(db.DateTime)

      @classmethod

      def create(cls, session, conf, name, stream, version, modulemd, scmurl, username,

          now = datetime.utcnow()

          module = cls(

              name=name,

              time_submitted=now,

              time_modified=now,

              # If the rebuild_strategy isn't specified, use the default

          )

          # Add a state transition to "init"

          mbt = ModuleBuildTrace(state_time=now, state=module.state)

              buildrequires = xmd['mbs']['buildrequires']

          except KeyError:

              buildrequires = {}

              'component_builds': [build.id for build in self.component_builds],

              'koji_tag': self.koji_tag,

              'owner': self.owner,

              'rebuild_strategy': self.rebuild_strategy,

              'scmurl': self.scmurl,

              'siblings': self.siblings,

              'state_reason': self.state_reason,

              'time_completed': _utc_datetime_to_iso(self.time_completed),

              'buildrequires': buildrequires,

          })

          if show_tasks:

      def extended_json(self, show_state_url=False, api_version=1):

          """

          SQLAlchemy sessions.

          :kwarg api_version: the API version to use when building the state URL

          """

          state_url = None

          if show_state_url:

              state_url = get_url_for('module_build', api_version=api_version, id=self.id)

              'base_module_buildrequires': [br.short_json(True) for br in self.buildrequires],

              'build_context': self.build_context,

              'modulemd': self.modulemd,

              'state_url': state_url,

          })

      def tasks(self):

          """

          done = (

              module_build_service.models.BUILD_STATES["failed"],

              module_build_service.models.BUILD_STATES["ready"],

              module_build_service.models.BUILD_STATES["done"],

          )

          result = module.state in done

      """Called whenever a module enters the 'done' state.

      We currently don't do anything useful, so moving to ready.

      Otherwise the done -> ready state should happen when all

      dependent modules were re-built, at least that's the current plan.

      """

          # This is ok.. it's a race condition we can ignore.

          pass

      build_logs.stop(build)

      module_build_service.builder.GenericBuilder.clear_cache(build)

      """

      log.info('Getting tag for %s:%s:%s', build.name, build.stream, build.version)

      if conf.system in ['koji', 'test']:

      else:

          return '-'.join(['module', build.name, build.stream, build.version])

      log.debug("Assigning koji tag=%s to module build" % tag)

      build.koji_tag = tag

          cg_build_koji_tag = get_content_generator_build_koji_tag(build_deps)

          log.debug("Assigning Content Generator build koji tag=%s to module build",

                    cg_build_koji_tag)

      return state

      """Generate a koji tag for a module

      Generally, a module's koji tag is in format ``module-N-S-V-C``. However, if

      :param str stream: a module's stream

      :param str version: a module's version

      :param str context: a module's context

          falling back to the old format of "module-<hash>". Default is 256

          characters, which is the maximum length of a tag Koji accepts.

      :return: a Koji tag

      :rtype: str

      """

      nsvc_list = [name, stream, str(version), context]

      if len(nsvc_tag) + len('-build') > max_length:

          # Fallback to the old format of 'module-<hash>' if the generated koji tag

          # name is longer than max_length

          nsvc_hash = hashlib.sha1('.'.join(nsvc_list).encode('utf-8')).hexdigest()[:16]

      return nsvc_tag

              log.warning('Module build {0} does not buildrequire a base module ({1})'

                          .format(module_build.id, ' or '.join(conf.base_module_names)))

      return '{prefix}{base_module_stream}{index}+{dist_hash}'.format(

          base_module_stream=base_module_stream,

          index=index,

          dist_hash=dist_hash,

      previous_module_build = session.query(models.ModuleBuild)\

          .filter_by(name=mmd.get_name())\

          .filter_by(stream=mmd.get_stream())\

          .filter(models.ModuleBuild.scmurl.isnot(None))\

          .filter_by(build_context=module.build_context)\

          .order_by(models.ModuleBuild.time_completed.desc())

  # Written by Ralph Bean <rbean@redhat.com>

  #            Matt Prahl <mprahl@redhat.com>

  #            Jan Kaluza <jkaluza@redhat.com>

  import re

  import time

  import shutil

      mmd.set_xmd(glib.dict_values(xmd))

  def record_component_builds(mmd, module, initial_batch=1,

                              previous_buildorder=None, main_mmd=None, session=None):

      # Imported here to allow import of utils in GenericBuilder.

      if not all_components:

          return

      rpm_weights = module_build_service.builder.GenericBuilder.get_build_weights(

          [c.get_name() for c in rpm_components])

      all_components.sort(key=lambda x: x.get_buildorder())

                                              previous_buildorder, main_mmd, session=session)

              continue

          # Skip the ComponentBuild if it already exists in database. This can happen

          # in case of module build resubmition.

          existing_build = models.ComponentBuild.from_component_name(

          if existing_build:

              # Check that the existing build has the same most important attributes.

              # This should never be a problem, but it's good to be defensive here so

          build = models.ComponentBuild(

              module_id=module.id,

              format="rpms",

              scmurl=full_url,

              batch=batch,

              ref=component_ref,

          )

          session.add(build)

      pagination_metadata, filter_module_builds, filter_component_builds,

      submit_module_build_from_scm, submit_module_build_from_yaml,

      get_scm_url_re, cors_header, validate_api_version, import_mmd,

  from module_build_service.errors import (

      ValidationError, Forbidden, NotFound, ProgrammingError)

  from module_build_service.backports import jsonify

      # Additional POST and DELETE handlers for modules follow.

      @validate_api_version()

      def post(self, api_version):

              handler = YAMLFileHandler(request)

          else:

              handler = SCMHandler(request)

  class BaseHandler(object):

      def __init__(self, request):

          self.username, self.groups = module_build_service.auth.get_user(request)

      @property

      def optional_params(self):

  class SCMHandler(BaseHandler):

      def validate(self, skip_branch=False, skip_optional_params=False):

          if "scmurl" not in self.data:

              log.error('Missing scmurl')

          if not conf.yaml_submit_allowed:

              raise Forbidden("YAML submission is not enabled")

          super(YAMLFileHandler, self).__init__(request)

      def validate(self):

          if "yaml" not in request.files:

            RESOLVER = 'db'

            # This is a whitelist of prefixes of koji tags we're allowed to manipulate

            DEFAULT_DIST_TAG_PREFIX = 'module'

          import_mmd(db.session, mmd)

      """

      Creates data_size * 3 modules in database in different states and

      with different component builds. See _populate_data for more info.

              mmd.set_stream(stream)

              import_mmd(db.session, mmd)

      with make_session(conf) as session:

      num_contexts = 2 if contexts else 1

      for index in range(data_size):

          for context in range(num_contexts):

              build_one.stream = '1'

              build_one.version = 2 + index

              build_one.state = BUILD_STATES['ready']

              if contexts:

                  build_one.stream = str(index)

                  unique_hash = hashlib.sha1(("%s:%s:%d:%d" % (

                  combined_hashes = '{0}:{1}'.format(unique_hash, unique_hash)

                  build_one.context = hashlib.sha1(combined_hashes.encode("utf-8")).hexdigest()[:8]

              build_one.modulemd = read_staged_data('nginx_mmd')

              build_one.scmurl = ('git://pkgs.domain.local/modules/nginx?'

                                  '#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')

              build_one.batch = 2

          build_two.stream = '1'

          build_two.version = 2 + index

          build_two.state = BUILD_STATES['done']

          build_two.modulemd = read_staged_data('testmodule')

          build_two.scmurl = ('git://pkgs.domain.local/modules/postgressql?'

                              '#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9')

          build_two.batch = 2

          build_three.stream = '4.3.43'

          build_three.version = 6 + index

          build_three.state = BUILD_STATES['wait']

          build_three.modulemd = read_staged_data('testmodule')

          build_three.koji_tag = None

          build_three.scmurl = ('git://pkgs.domain.local/modules/testmodule?'

          session.commit()

      """ Creates a testmodule in the building state with all the components in the same batch

      """

      clean_database()

      build_one.stream = 'master'

      build_one.version = 20170109091357

      build_one.state = BUILD_STATES['build']

      build_one.build_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'

      build_one.runtime_context = 'ac4de1c346dcf09ce77d38cd4e75094ec1c08eb0'

      build_one.context = '7c29193d'

      build_one.scmurl = 'https://src.stg.fedoraproject.org/modules/testmodule.git?#ff1ea79'

      if tangerine_state:

          build_one.batch = 3

          release = module_build_service.utils.get_rpm_release(build_one)

          assert release == 'module+f28+2+814cfa39'

      @pytest.mark.parametrize('scmurl', [

          ('https://src.stg.fedoraproject.org/modules/testmodule.git'

           '?#620ec77321b2ea7b0d67d82992dda3e1d67055b4'),

  import module_build_service.scm

  from mock import patch, PropertyMock

  from shutil import copyfile

  from os import path, mkdir

  from os.path import dirname

          assert data['context'] == '00000000'

          assert data['name'] == 'nginx'

          assert data['owner'] == 'Moe Szyslak'

          assert data['stream'] == '1'

          assert data['siblings'] == []

          assert data['state'] == 5

          assert data['owner'] == 'Moe Szyslak'

          assert data['scmurl'] == ('git://pkgs.domain.local/modules/nginx'

                                    '?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9')

          assert data['siblings'] == []

          assert data['state'] == 5

          assert data['state_name'] == 'ready'

                  "rebuild_strategy": "changed-and-after",

                  "scmurl": ("git://pkgs.domain.local/modules/testmodule"

                             "?#ca95886c7a443b36a9ce31abda1f9bef22f2f8c9"),

                  "siblings": [],

                  "state": 1,

                  "state_name": "wait",

                  "state_reason": None,

                  "rebuild_strategy": "changed-and-after",

                  "scmurl": ("git://pkgs.domain.local/modules/postgressql"

                             "?#aa95886c7a443b36a9ce31abda1f9bef22f2f8c9"),

                  "siblings": [],

                  "state": 3,

                  "state_name": "done",

                  "state_reason": None,

                  "rebuild_strategy": "changed-and-after",

                  "scmurl": ("git://pkgs.domain.local/modules/nginx"

                             "?#ba95886c7a443b36a9ce31abda1f9bef22f2f8c9"),

                  "siblings": [2],

                  "state": 5,

                  "state_name": "ready",

                  "state_reason": None,

          assert br_modulea == buildrequires.get('modulea')

          assert br_moduleb == buildrequires.get('moduleb')