#554 Modify content generator based on Koji dev feedback
Merged 5 years ago by jkaluza. Opened 5 years ago by sochotni.
sochotni/fm-orchestrator content-generators  into  master

file modified
+8
@@ -484,3 +484,11 @@ 

  - if `MODULE_BUILD_SERVICE_DEVELOPER_ENV` is set to some reasonable

    value, DevConfiguration is forced and `config.py` is used directly from the

    MBS's develop instance. For more information see `docs/CONTRIBUTING.rst`.

+ 

+ License

+ =======

+ 

+ MBS is licensed under MIT license. See LICENSE file for details.

+ 

+ Parts of MBS are licensed under 3-clause BSD license from:

+ https://github.com/projectatomic/atomic-reactor/blob/master/LICENSE

@@ -27,8 +27,10 @@ 

  import logging

  import json

  import os

+ import pkg_resources

  import platform

  import shutil

+ import subprocess

  import tempfile

  

  import koji
@@ -59,6 +61,122 @@ 

      def __repr__(self):

          return "<KojiContentGenerator module: %s>" % (self.module_name)

  

+     @staticmethod

+     def parse_rpm_output(output, tags, separator=';'):

+         """

+         Copied from https://github.com/projectatomic/atomic-reactor/blob/master/atomic_reactor/plugins/exit_koji_promote.py

+         License: BSD 3-clause

+ 

+         Parse output of the rpm query.

+         :param output: list, decoded output (str) from the rpm subprocess

+         :param tags: list, str fields used for query output

+         :return: list, dicts describing each rpm package

+         """

+ 

+         def field(tag):

+             """

+             Get a field value by name

+             """

+             try:

+                 value = fields[tags.index(tag)]

+             except ValueError:

+                 return None

+ 

+             if value == '(none)':

+                 return None

+ 

+             return value

+ 

+         components = []

+         sigmarker = 'Key ID '

+         for rpm in output:

+             fields = rpm.rstrip('\n').split(separator)

+             if len(fields) < len(tags):

+                 continue

+ 

+             signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')

+             if signature:

+                 parts = signature.split(sigmarker, 1)

+                 if len(parts) > 1:

+                     signature = parts[1]

+ 

+             component_rpm = {

+                 'type': 'rpm',

+                 'name': field('NAME'),

+                 'version': field('VERSION'),

+                 'release': field('RELEASE'),

+                 'arch': field('ARCH'),

+                 'sigmd5': field('SIGMD5'),

+                 'signature': signature,

+             }

+ 

+             # Special handling for epoch as it must be an integer or None

+             epoch = field('EPOCH')

+             if epoch is not None:

+                 epoch = int(epoch)

+ 

+             component_rpm['epoch'] = epoch

+ 

+             if component_rpm['name'] != 'gpg-pubkey':

+                 components.append(component_rpm)

+ 

+         return components

+ 

+     def __get_rpms(self):

+         """

+         Copied from https://github.com/projectatomic/atomic-reactor/blob/master/atomic_reactor/plugins/exit_koji_promote.py

+         License: BSD 3-clause

+ 

+         Build a list of installed RPMs in the format required for the

+         metadata.

+         """

+ 

+         tags = [

+             'NAME',

+             'VERSION',

+             'RELEASE',

+             'ARCH',

+             'EPOCH',

+             'SIGMD5',

+             'SIGPGP:pgpsig',

+             'SIGGPG:pgpsig',

+         ]

+ 

+         sep = ';'

+         fmt = sep.join(["%%{%s}" % tag for tag in tags])

+         cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)

+         try:

+             # py3

+             (status, output) = subprocess.getstatusoutput(cmd)

+         except AttributeError:

+             # py2

+             with open('/dev/null', 'r+') as devnull:

+                 p = subprocess.Popen(cmd,

+                                      shell=True,

+                                      stdin=devnull,

+                                      stdout=subprocess.PIPE,

+                                      stderr=devnull)

+ 

+                 (stdout, stderr) = p.communicate()

+                 status = p.wait()

+                 output = stdout.decode()

+ 

+         if status != 0:

+             log.debug("%s: stderr output: %s", cmd, stderr)

+             raise RuntimeError("%s: exit code %s" % (cmd, status))

+ 

+         return self.parse_rpm_output(output.splitlines(), tags, separator=sep)

+ 

+     def __get_tools(self):

+         """Return list of tools which are important for reproducing mbs outputs"""

+ 

+         tools = ["modulemd"]

+         ret = []

+         for tool in tools:

+             ret.append({"name": tool,

+                         "version": pkg_resources.get_distribution(tool).version})

+         return ret

+ 

      def _koji_rpms_in_tag(self, tag):

          """ Return the list of koji rpms in a tag. """

          log.debug("Listing rpms in koji tag %s", tag)
@@ -83,7 +201,7 @@ 

      def _get_build(self):

          ret = {}

          ret['name'] = self.module.name

-         ret['version'] = self.module.stream

+         ret['version'] = self.module.stream.replace("-", "_")

          ret['release'] = self.module.version

          ret['source'] = self.module.scmurl

          ret['start_time'] = calendar.timegm(
@@ -92,16 +210,18 @@ 

              self.module.time_completed.utctimetuple())

          ret['extra'] = {

              "typeinfo": {

-                 "modulemd": {

+                 "module": {

                      "module_build_service_id": self.module.id,

-                     "modulemd_str": self.module.modulemd

+                     "modulemd_str": self.module.modulemd,

+                     "name": self.module.name,

+                     "stream": self.module.stream,

+                     "version": self.module.version

                  }

              }

          }

          return ret

  

      def _get_buildroot(self):

-         import pkg_resources

          version = pkg_resources.get_distribution("module-build-service").version

          distro = platform.linux_distribution()

          ret = {
@@ -118,13 +238,12 @@ 

                  "arch": platform.machine(),

                  "type": "none"

              },

-             "components": [],

-             "tools": []

+             "components": self.__get_rpms(),

+             "tools": self.__get_tools()

          }

          return ret

  

  

- 

      def _get_output(self):

          ret = []

          rpms = self._koji_rpms_in_tag(self.module.koji_tag)
@@ -145,8 +264,13 @@ 

          ret.append(

              {

                  'buildroot_id': 1,

-                 'arch': "noarch",

-                 'type': 'modulemd',

+                 'arch': 'noarch',

+                 'type': 'file',

+                 'extra': {

+                     'typeinfo': {

+                         'module': {}

+                     }

+                 },

                  'filesize': len(self.mmd),

                  'checksum_type': 'md5',

                  'checksum': hashlib.md5(self.mmd).hexdigest(),

@@ -69,17 +69,25 @@ 

          import moksha.hub.reactor

          self.vcr.__exit__()

  

+     @patch("subprocess.Popen")

      @patch("pkg_resources.get_distribution")

      @patch("platform.linux_distribution")

      @patch("platform.machine")

      @patch("module_build_service.builder.KojiContentGenerator.KojiContentGenerator._koji_rpms_in_tag")

-     def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res):

+     def test_get_generator_json(self, rpms_in_tag, machine, distro, pkg_res, popen):

          """ Test generation of content generator json """

          self.maxDiff = None

          distro.return_value = ("Fedora", "25", "Twenty Five")

          machine.return_value = "i686"

          pkg_res.return_value = Mock()

          pkg_res.return_value.version = "current-tested-version"

+         rpm_mock = Mock()

+         rpm_out =  "rpm-name;1.0;r1;x86_64;(none);sigmd5:1;sigpgp:p;siggpg:g\n" \

+                    "rpm-name-2;2.0;r2;i686;1;sigmd5:2;sigpgp:p2;siggpg:g2"

+         attrs = {'communicate.return_value': (rpm_out, 'error'),

+                  'wait.return_value': 0}

+         rpm_mock.configure_mock(**attrs)

+         popen.return_value = rpm_mock

  

          tests_dir = path.abspath(path.dirname(__file__))

          rpm_in_tag_path = path.join(tests_dir,

@@ -9,8 +9,34 @@ 

              "name": "module-build-service",

              "version": "current-tested-version"

          },

-         "tools": [],

-         "components": [],

+         "tools": [

+             {

+                 "name": "modulemd",

+                 "version": "current-tested-version"

+             }

+         ],

+         "components": [

+             {

+                 "name": "rpm-name",

+                 "version": "1.0",

+                 "release": "r1",

+                 "epoch": null,

+                 "arch": "x86_64",

+                 "sigmd5": "sigmd5:1",

+                 "signature": "sigpgp:p",

+                 "type": "rpm"

+             },

+             {

+                 "name": "rpm-name-2",

+                 "version": "2.0",

+                 "release": "r2",

+                 "epoch": 1,

+                 "arch": "i686",

+                 "sigmd5": "sigmd5:2",

+                 "signature": "sigpgp:p2",

+                 "type": "rpm"

+             }

+         ],

          "container": {

              "arch": "i686",

              "type": "none"
@@ -602,7 +628,12 @@ 

              "filesize": 1134,

              "checksum": "bf1615b15f6a0fee485abe94af6b56b6",

              "checksum_type": "md5",

-             "type": "modulemd"

+             "type": "file",

+             "extra": {

+                 "typeinfo": {

+                     "module": {}

+                 }

+             }

          }

      ],

      "metadata_version": 0,
@@ -613,7 +644,10 @@ 

          "release": "2",

          "extra": {

              "typeinfo": {

-                 "modulemd": {

+                 "module": {

+                     "name": "nginx",

+                     "stream": "1",

+                     "version": "2",

                      "module_build_service_id": 1,

                      "modulemd_str": "# Document type identifier\ndocument: modulemd\n# Module metadata format version\nversion: 1\ndata:\n    # Module name, optional\n    # Typically filled in by the buildsystem, using the VCS repository\n    # name as the name of the module.\n    name: nginx\n    # Module update stream, optional\n    # Typically filled in by the buildsystem, using the VCS branch name\n    # as the name of the stream.\n    stream: 1\n    # Module version, integer, optional, cannot be negative\n    # Typically filled in by the buildsystem, using the VCS commit\n    # timestamp.  Module version defines upgrade path for the particular\n    # update stream.\n    version: 2\n    # A short summary describing the module, required\n    summary: An example nginx module\n    # A verbose description of the module, required\n    description: >\n        A module for the tests of module build service\n    # Module and content licenses in the Fedora license identifier\n    # format, required\n    license:\n        # Module license, required\n        # This list covers licenses used for the module metadata, SPEC\n        # files or extra patches\n        module:\n            - MIT\n"

                  }

  • Change the type of build from modulemd to just module
  • Change build output type from modulemd to file
  • Provide the typeinfo also for the modulemd.yaml output
  • Convert dashes to underscores for build version (i.e. mbs stream).
    koji build versions can't have dashes - we can provide real value in
    the extra section
  • Add name, stream, version data into build extra section
  • Add tool information for modulemd
  • Add buildroot components from host running mbs
  • Added few methods licensed under BSD 3-clause (from atomic-reactor)

Pull-Request has been merged by jkaluza

5 years ago