#228 Add multilib_arches and multilib_method.
Merged 10 months ago by jkaluza. Opened 10 months ago by jkaluza.
jkaluza/odcs multilib-arches  into  master

file modified
+6

@@ -138,6 +138,12 @@ 

      - `iso` - Generates non-installable ISO files with RPMs from a compose.

      - `boot.iso` - Generates `images/boot.iso` file which is needed to build base container images from resulting compose.

  - `arches` - List of additional Koji arches to build this compose for. By default, the compose is built only for "x86_64" arch.

+ - `multilib_arches` - Subset of `arches` for which the multilib should be enabled. For each architecture in the `multilib_arches` list, ODCS will include also packages from other compatible architectures in a compose. For example when "x86_64" is included `multilib_arches`, ODCS will include also "i686" packages in a compose. The set of packages included in a composes is influenced by `multilib_method` list.

+ - `multilib_method` - List defining the method used to determine whether consider package as multilib. Defaults to empty list. The list can have following values:

+     - `iso` - Generates non-installable ISO files with RPMs from a compose.

+     - `runtime` - Packages whose name ends with "-devel" or "-static" suffix will be considered as multilib. 

+     - `devel` - Packages that install some shared object file "*.so.*" will be considered as multilib. 

+     - `all` - All pakages will be considered as multilib.

  

  The `new_compose` method returns `dict` object describing the compose, for example:

  

@@ -61,6 +61,20 @@ 

  

  INVERSE_COMPOSE_STATES = {v: k for k, v in COMPOSE_STATES.items()}

  

+ MULTILIB_METHODS = {

+     "none": 0,

+     # Packages whose name ends with "-devel" or "-static" suffix will

+     # be considered as multilib.

+     "runtime": 1,

+     # Packages that install some shared object file "*.so.*" will be

+     # considered as multilib.

+     "devel": 2,

+     # All pakages will be considered as multilib.

+     "all": 4,

+ }

+ 

+ INVERSE_MULTILIB_METHODS = {v: k for k, v in MULTILIB_METHODS.items()}

+ 

  COMPOSE_RESULTS = {

      "repository": 1,

      "iso": 2,

file modified
+8

@@ -43,6 +43,14 @@ 

  

  filter_system_release_packages = False

  

+ multilib = [

+     ('^.*$', {

+ {%- for multilib_arch in config.multilib_arches %}

+         '{{ multilib_arch }}': {{ config.multilib_method }}

+ {%- endfor%}

+     }),

+ ]

+ 

  # GATHER

  gather_source = '{{ config.gather_source }}'

  gather_method = '{{ config.gather_method }}'

file modified
+19 -1

@@ -381,6 +381,22 @@ 

                        old_compose)

              continue

  

+         multilib_arches = set(compose.multilib_arches.split(" ")) \

+             if compose.multilib_arches else set()

+         old_multilib_arches = set(old_compose.multilib_arches.split(" ")) \

+             if old_compose.multilib_arches else set()

+         if multilib_arches != old_multilib_arches:

+             log.debug("%r: Cannot reuse %r - multilib_arches not same", compose,

+                       old_compose)

+             continue

+ 

+         multilib_method = compose.multilib_method

+         old_multilib_method = old_compose.multilib_method

+         if multilib_method != old_multilib_method:

+             log.debug("%r: Cannot reuse %r - multilib_method not same", compose,

+                       old_compose)

+             continue

+ 

          # In case of compose renewal, the compose.koji_event will be actually

          # lower than the "old_compose"'s one - the `compose` might have been for

          # example submitted 1 year ago, so koji_event will be one year old.

@@ -538,7 +554,9 @@ 

                                      compose.source, packages=packages,

                                      sigkeys=compose.sigkeys,

                                      results=compose.results,

-                                     arches=compose.arches.split(" "))

+                                     arches=compose.arches.split(" "),

+                                     multilib_arches=compose.multilib_arches.split(" "),

+                                     multilib_method=compose.multilib_method)

              if compose.flags & COMPOSE_FLAGS["no_deps"]:

                  pungi_cfg.gather_method = "nodeps"

              if compose.flags & COMPOSE_FLAGS["no_inheritance"]:

@@ -0,0 +1,26 @@ 

+ """Add multilib_arches and multilib_method

+ 

+ Revision ID: d1da07e15c54

+ Revises: f4bc999818d7

+ Create Date: 2018-09-03 14:06:40.565612

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = 'd1da07e15c54'

+ down_revision = 'f4bc999818d7'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     op.add_column('composes', sa.Column('multilib_arches', sa.String(), nullable=True,

+                                         default=""))

+     op.add_column('composes', sa.Column('multilib_method', sa.Integer(), nullable=True,

+                                         default=0))

+ 

+ 

+ def downgrade():

+     op.drop_column('composes', 'multilib_method')

+     op.drop_column('composes', 'multilib_arches')

file modified
+12 -2

@@ -130,11 +130,15 @@ 

      koji_task_id = db.Column(db.Integer, index=True)

      # White-space separated list of arches to build for.

      arches = db.Column(db.String)

+     # White-space separated list of arches to enable multilib for.

+     multilib_arches = db.Column(db.String)

+     # Method to generate multilib compose as defined by python-multilib.

+     multilib_method = db.Column(db.Integer)

  

      @classmethod

      def create(cls, session, owner, source_type, source, results,

                 seconds_to_live, packages=None, flags=0, sigkeys=None,

-                arches=None):

+                arches=None, multilib_arches=None, multilib_method=None):

          now = datetime.utcnow()

          compose = cls(

              owner=owner,

@@ -147,7 +151,9 @@ 

              time_to_expire=now + timedelta(seconds=seconds_to_live),

              packages=packages,

              flags=flags,

-             arches=arches if arches else " ".join(conf.arches)

+             arches=arches if arches else " ".join(conf.arches),

+             multilib_arches=multilib_arches if multilib_arches else "",

+             multilib_method=multilib_method if multilib_method else 0

          )

          session.add(compose)

          return compose

@@ -175,6 +181,8 @@ 

              flags=compose.flags,

              koji_event=compose.koji_event,

              arches=compose.arches,

+             multilib_arches=compose.multilib_arches,

+             multilib_method=compose.multilib_method,

          )

          session.add(compose)

          return compose

@@ -292,6 +300,8 @@ 

              'koji_task_id': self.koji_task_id,

              'packages': self.packages,

              'arches': self.arches,

+             'multilib_arches': self.multilib_arches,

+             'multilib_method': self.multilib_method,

          }

  

      @staticmethod

file modified
+9 -2

@@ -34,7 +34,7 @@ 

  import odcs.server.utils

  from odcs.server import conf, log, db

  from odcs.server import comps

- from odcs.common.types import PungiSourceType, COMPOSE_RESULTS

+ from odcs.common.types import PungiSourceType, COMPOSE_RESULTS, MULTILIB_METHODS

  from odcs.server.utils import makedirs, clone_repo, copytree

  

  

@@ -109,7 +109,8 @@ 

  

  class PungiConfig(BasePungiConfig):

      def __init__(self, release_name, release_version, source_type, source,

-                  packages=None, arches=None, sigkeys=None, results=0):

+                  packages=None, arches=None, sigkeys=None, results=0,

+                  multilib_arches=None, multilib_method=0):

          self.release_name = release_name

          self.release_version = release_version

          self.bootable = False

@@ -131,6 +132,12 @@ 

              if results & v:

                  self.results.append(k)

  

+         self.multilib_arches = multilib_arches if multilib_arches else []

+         self.multilib_method = []

+         for k, v in MULTILIB_METHODS.items():

+             if multilib_method & v:

+                 self.multilib_method.append(k)

+ 

          if "boot.iso" in self.results:

              self.bootable = True

  

file modified
+14 -2

@@ -32,7 +32,7 @@ 

  from odcs.server.models import Compose

  from odcs.common.types import (

      COMPOSE_RESULTS, COMPOSE_FLAGS, COMPOSE_STATES, PUNGI_SOURCE_TYPE_NAMES,

-     PungiSourceType)

+     PungiSourceType, MULTILIB_METHODS)

  from odcs.server.api_utils import (

      pagination_metadata, filter_composes, validate_json_data,

      raise_if_input_not_allowed)

@@ -284,6 +284,17 @@ 

          else:

              arches = " ".join(conf.arches)

  

+         multilib_arches = ""

+         if "multilib_arches" in data:

+             multilib_arches = " ".join(data["multilib_arches"])

+ 

+         multilib_method = MULTILIB_METHODS["none"]

+         if "multilib_method" in data:

+             for name in data["multilib_method"]:

+                 if name not in MULTILIB_METHODS:

+                     raise ValueError("Unknown multilib method \"%s\"" % name)

+                 multilib_method |= MULTILIB_METHODS[name]

+ 

          raise_if_input_not_allowed(

              source_types=source_type, sources=source, results=results,

              flags=flags, arches=arches)

@@ -291,7 +302,8 @@ 

          compose = Compose.create(

              db.session, self._get_compose_owner(), source_type, source,

              results, seconds_to_live,

-             packages, flags, sigkeys, arches)

+             packages, flags, sigkeys, arches, multilib_arches=multilib_arches,

+             multilib_method=multilib_method)

          db.session.add(compose)

          # Flush is needed, because we use `before_commit` SQLAlchemy event to

          # send message and before_commit can be called before flush and

file modified
+10 -4

@@ -316,14 +316,17 @@ 

          attrs["koji_event"] = 123456

          attrs["source"] = "123"

          attrs["arches"] = "ppc64 x86_64"

+         attrs["multilib_arches"] = "x86_64 i686"

+         attrs["multilib_method"] = 1

          for attr, value in attrs.items():

              c = Compose.create(

                  db.session, "me", PungiSourceType.REPO, os.path.join(thisdir, "repo"),

-                 COMPOSE_RESULTS["repository"], 3600, packages="ed")

+                 COMPOSE_RESULTS["repository"], 3600, packages="ed", sigkeys="123")

              setattr(c, attr, value)

  

-             # Do not resolve compose for non-existing source...

-             if attr != "source":

+             # Do not resolve compose for non-existing source and in case we

+             # change koji_event, because it would be overwriten.

+             if attr not in ["source", "koji_event"]:

                  resolve_compose(c)

  

              db.session.add(c)

@@ -571,7 +574,8 @@ 

          c = Compose.create(

              db.session, "me", PungiSourceType.KOJI_TAG, "f26",

              COMPOSE_RESULTS["repository"], 60, packages='pkg1 pkg2 pkg3',

-             arches="x86_64 s390")

+             arches="x86_64 s390", multilib_arches="i686 x86_64",

+             multilib_method=1)

          c.id = 1

  

          generate_pungi_compose(c)

@@ -587,6 +591,8 @@ 

          self.assertEqual(self.pungi_config.gather_method, "deps")

          self.assertEqual(self.pungi_config.pkgset_koji_inherit, True)

          self.assertEqual(set(self.pungi_config.arches), set(["x86_64", "s390"]))

+         self.assertEqual(set(self.pungi_config.multilib_arches), set(["i686", "x86_64"]))

+         self.assertEqual(self.pungi_config.multilib_method, ["runtime"])

  

      def test_generate_pungi_compose_nodeps(self):

          c = Compose.create(

file modified
+3 -1

@@ -66,7 +66,9 @@ 

                           'koji_event': None,

                           'koji_task_id': None,

                           'packages': None,

-                          'arches': 'x86_64'}

+                          'arches': 'x86_64',

+                          'multilib_arches': '',

+                          'multilib_method': 0}

          self.assertEqual(c.json(), expected_json)

  

  

@@ -156,6 +156,24 @@ 

              cfg = self._load_pungi_cfg(template)

              self.assertTrue(cfg["pkgset_koji_inherit"])

  

+     def test_get_pungi_conf_multilib(self):

+         _, mock_path = tempfile.mkstemp()

+         template_path = os.path.abspath(os.path.join(test_dir,

+                                                      "../conf/pungi.conf"))

+         shutil.copy2(template_path, mock_path)

+ 

+         with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):

+             pungi_cfg = PungiConfig("MBS-512", "1", PungiSourceType.KOJI_TAG,

+                                     "f26", multilib_arches=["x86_64", "s390x"],

+                                     multilib_method=3)

+ 

+             template = pungi_cfg.get_pungi_config()

+             cfg = self._load_pungi_cfg(template)

+             self.assertEqual(set(cfg["multilib"][0][1].keys()), set(["s390x", "x86_64"]))

+             for variant, arch_method_dict in cfg["multilib"]:

+                 for method in arch_method_dict.values():

+                     self.assertEqual(set(method), set(['runtime', 'devel']))

+ 

  

  class TestPungi(unittest.TestCase):

  

file modified
+57 -3

@@ -35,7 +35,8 @@ 

  

  from odcs.server import conf, db, app, login_manager, version

  from odcs.server.models import Compose, User

- from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS, COMPOSE_FLAGS

+ from odcs.common.types import (COMPOSE_STATES, COMPOSE_RESULTS, COMPOSE_FLAGS,

+                                MULTILIB_METHODS)

  from odcs.server.pungi import PungiSourceType

  from .utils import ModelsBaseTest

  from odcs.server.api_utils import validate_json_data

@@ -299,7 +300,9 @@ 

                           'koji_event': None,

                           'koji_task_id': None,

                           'packages': None,

-                          'arches': 'x86_64'}

+                          'arches': 'x86_64',

+                          'multilib_arches': '',

+                          'multilib_method': 0}

          self.assertEqual(data, expected_json)

  

          db.session.expire_all()

@@ -429,6 +432,55 @@ 

          c = db.session.query(Compose).filter(Compose.id == 1).one()

          self.assertEqual(c.state, COMPOSE_STATES["wait"])

  

+     def test_submit_build_multilib_arches(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/1/composes/', data=json.dumps(

+                 {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']},

+                  'arches': ["ppc64", "s390"], 'multilib_arches': ["x86_64", "ppc64le"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['multilib_arches'], 'x86_64 ppc64le')

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_multilib_method(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/1/composes/', data=json.dumps(

+                 {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']},

+                  'arches': ["ppc64", "s390"], 'multilib_method': ["runtime", "devel"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(data['multilib_method'],

+                          MULTILIB_METHODS["runtime"] | MULTILIB_METHODS["devel"])

+ 

+         db.session.expire_all()

+         c = db.session.query(Compose).filter(Compose.id == 1).one()

+         self.assertEqual(c.state, COMPOSE_STATES["wait"])

+ 

+     def test_submit_build_multilib_method_unknown(self):

+         with self.test_request_context(user='dev'):

+             flask.g.oidc_scopes = [

+                 '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')

+             ]

+ 

+             rv = self.client.post('/api/1/composes/', data=json.dumps(

+                 {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']},

+                  'arches': ["ppc64", "s390"], 'multilib_method': ["foo", "devel"]}))

+             data = json.loads(rv.get_data(as_text=True))

+ 

+         self.assertEqual(

+             data['message'], 'Unknown multilib method "foo"')

+ 

      def test_submit_build_duplicate_sources(self):

          with self.test_request_context(user='dev'):

              flask.g.oidc_scopes = [

@@ -898,7 +950,9 @@ 

                           'koji_event': None,

                           'koji_task_id': None,

                           'packages': None,

-                          'arches': 'x86_64'}

+                          'arches': 'x86_64',

+                          'multilib_arches': '',

+                          'multilib_method': 0}

          self.assertEqual(data, expected_json)

  

          db.session.expire_all()

This PR allows generating "multilib" composes - these are composes where for example x86_64 repository contains also i686 packages.

There is Pungi multilib configuration which enables that and allows further configuration: https://pagure.io/pungi/blob/master/f/doc/configuration.rst?text=True#_653.

This PR adds new multilib_arches and multilib_method REST API POST fields which can be used to configure the Pungi multilib value. This is why the PR changes views.py and also models.py, because we need to store the new fields in DB.

It also changes the get_reusable_compose code to not reuse composes with different multilib settings.

Pull-Request has been merged by jkaluza

10 months ago