PR#228: Add multilib_arches and multilib_method. - odcs

odcs

#228 Add multilib_arches and multilib_method.

Merged 5 years ago by jkaluza. Opened 5 years ago by jkaluza.

jkaluza/odcs multilib-arches into master

Add multilib_arches and multilib_method.

Jan Kaluza • 5 years ago

09b2cf7

README.md

file modified

		`@@ -138,6 +138,12 @@`
		- `iso` - Generates non-installable ISO files with RPMs from a compose.
		- `boot.iso` - Generates `images/boot.iso` file which is needed to build base container images from resulting compose.
		- `arches` - List of additional Koji arches to build this compose for. By default, the compose is built only for "x86_64" arch.
		+ - `multilib_arches` - Subset of `arches` for which the multilib should be enabled. For each architecture in the `multilib_arches` list, ODCS will include also packages from other compatible architectures in a compose. For example when "x86_64" is included `multilib_arches`, ODCS will include also "i686" packages in a compose. The set of packages included in a composes is influenced by `multilib_method` list.
		+ - `multilib_method` - List defining the method used to determine whether consider package as multilib. Defaults to empty list. The list can have following values:
		+ - `iso` - Generates non-installable ISO files with RPMs from a compose.
		+ - `runtime` - Packages whose name ends with "-devel" or "-static" suffix will be considered as multilib.
		+ - `devel` - Packages that install some shared object file ".so." will be considered as multilib.
		+ - `all` - All pakages will be considered as multilib.

		The `new_compose` method returns `dict` object describing the compose, for example:

common/odcs/common/types.py

file modified

+14

		`@@ -61,6 +61,20 @@`

		`INVERSE_COMPOSE_STATES = {v: k for k, v in COMPOSE_STATES.items()}`

		`+ MULTILIB_METHODS = {`
		`+ "none": 0,`
		`+ # Packages whose name ends with "-devel" or "-static" suffix will`
		`+ # be considered as multilib.`
		`+ "runtime": 1,`
		`+ # Packages that install some shared object file ".so." will be`
		`+ # considered as multilib.`
		`+ "devel": 2,`
		`+ # All pakages will be considered as multilib.`
		`+ "all": 4,`
		`+ }`
		`+`
		`+ INVERSE_MULTILIB_METHODS = {v: k for k, v in MULTILIB_METHODS.items()}`
		`+`
		`COMPOSE_RESULTS = {`
		`"repository": 1,`
		`"iso": 2,`

server/conf/pungi.conf

file modified

		`@@ -43,6 +43,14 @@`

		`filter_system_release_packages = False`

		`+ multilib = [`
		`+ ('^.*$', {`
		`+ {%- for multilib_arch in config.multilib_arches %}`
		`+ '{{ multilib_arch }}': {{ config.multilib_method }}`
		`+ {%- endfor%}`
		`+ }),`
		`+ ]`
		`+`
		`# GATHER`
		`gather_source = '{{ config.gather_source }}'`
		`gather_method = '{{ config.gather_method }}'`

server/odcs/server/backend.py

file modified

+19 -1

		`@@ -381,6 +381,22 @@`
		`old_compose)`
		`continue`

		`+ multilib_arches = set(compose.multilib_arches.split(" ")) \`
		`+ if compose.multilib_arches else set()`
		`+ old_multilib_arches = set(old_compose.multilib_arches.split(" ")) \`
		`+ if old_compose.multilib_arches else set()`
		`+ if multilib_arches != old_multilib_arches:`
		`+ log.debug("%r: Cannot reuse %r - multilib_arches not same", compose,`
		`+ old_compose)`
		`+ continue`
		`+`
		`+ multilib_method = compose.multilib_method`
		`+ old_multilib_method = old_compose.multilib_method`
		`+ if multilib_method != old_multilib_method:`
		`+ log.debug("%r: Cannot reuse %r - multilib_method not same", compose,`
		`+ old_compose)`
		`+ continue`
		`+`
		`# In case of compose renewal, the compose.koji_event will be actually`
		# lower than the "old_compose"'s one - the `compose` might have been for
		`# example submitted 1 year ago, so koji_event will be one year old.`
		`@@ -538,7 +554,9 @@`
		`compose.source, packages=packages,`
		`sigkeys=compose.sigkeys,`
		`results=compose.results,`
		`- arches=compose.arches.split(" "))`
		`+ arches=compose.arches.split(" "),`
		`+ multilib_arches=compose.multilib_arches.split(" "),`
		`+ multilib_method=compose.multilib_method)`
		`if compose.flags & COMPOSE_FLAGS["no_deps"]:`
		`pungi_cfg.gather_method = "nodeps"`
		`if compose.flags & COMPOSE_FLAGS["no_inheritance"]:`

server/odcs/server/migrations/versions/d1da07e15c54_.py

file added

+26

		`@@ -0,0 +1,26 @@`
		`+ """Add multilib_arches and multilib_method`
		`+`
		`+ Revision ID: d1da07e15c54`
		`+ Revises: f4bc999818d7`
		`+ Create Date: 2018-09-03 14:06:40.565612`
		`+`
		`+ """`
		`+`
		`+ # revision identifiers, used by Alembic.`
		`+ revision = 'd1da07e15c54'`
		`+ down_revision = 'f4bc999818d7'`
		`+`
		`+ from alembic import op`
		`+ import sqlalchemy as sa`
		`+`
		`+`
		`+ def upgrade():`
		`+ op.add_column('composes', sa.Column('multilib_arches', sa.String(), nullable=True,`
		`+ default=""))`
		`+ op.add_column('composes', sa.Column('multilib_method', sa.Integer(), nullable=True,`
		`+ default=0))`
		`+`
		`+`
		`+ def downgrade():`
		`+ op.drop_column('composes', 'multilib_method')`
		`+ op.drop_column('composes', 'multilib_arches')`

server/odcs/server/models.py

file modified

+12 -2

		`@@ -130,11 +130,15 @@`
		`koji_task_id = db.Column(db.Integer, index=True)`
		`# White-space separated list of arches to build for.`
		`arches = db.Column(db.String)`
		`+ # White-space separated list of arches to enable multilib for.`
		`+ multilib_arches = db.Column(db.String)`
		`+ # Method to generate multilib compose as defined by python-multilib.`
		`+ multilib_method = db.Column(db.Integer)`

		`@classmethod`
		`def create(cls, session, owner, source_type, source, results,`
		`seconds_to_live, packages=None, flags=0, sigkeys=None,`
		`- arches=None):`
		`+ arches=None, multilib_arches=None, multilib_method=None):`
		`now = datetime.utcnow()`
		`compose = cls(`
		`owner=owner,`
		`@@ -147,7 +151,9 @@`
		`time_to_expire=now + timedelta(seconds=seconds_to_live),`
		`packages=packages,`
		`flags=flags,`
		`- arches=arches if arches else " ".join(conf.arches)`
		`+ arches=arches if arches else " ".join(conf.arches),`
		`+ multilib_arches=multilib_arches if multilib_arches else "",`
		`+ multilib_method=multilib_method if multilib_method else 0`
		`)`
		`session.add(compose)`
		`return compose`
		`@@ -175,6 +181,8 @@`
		`flags=compose.flags,`
		`koji_event=compose.koji_event,`
		`arches=compose.arches,`
		`+ multilib_arches=compose.multilib_arches,`
		`+ multilib_method=compose.multilib_method,`
		`)`
		`session.add(compose)`
		`return compose`
		`@@ -292,6 +300,8 @@`
		`'koji_task_id': self.koji_task_id,`
		`'packages': self.packages,`
		`'arches': self.arches,`
		`+ 'multilib_arches': self.multilib_arches,`
		`+ 'multilib_method': self.multilib_method,`
		`}`

		`@staticmethod`

server/odcs/server/pungi.py

file modified

+9 -2

		`@@ -34,7 +34,7 @@`
		`import odcs.server.utils`
		`from odcs.server import conf, log, db`
		`from odcs.server import comps`
		`- from odcs.common.types import PungiSourceType, COMPOSE_RESULTS`
		`+ from odcs.common.types import PungiSourceType, COMPOSE_RESULTS, MULTILIB_METHODS`
		`from odcs.server.utils import makedirs, clone_repo, copytree`


		`@@ -109,7 +109,8 @@`

		`class PungiConfig(BasePungiConfig):`
		`def __init__(self, release_name, release_version, source_type, source,`
		`- packages=None, arches=None, sigkeys=None, results=0):`
		`+ packages=None, arches=None, sigkeys=None, results=0,`
		`+ multilib_arches=None, multilib_method=0):`
		`self.release_name = release_name`
		`self.release_version = release_version`
		`self.bootable = False`
		`@@ -131,6 +132,12 @@`
		`if results & v:`
		`self.results.append(k)`

		`+ self.multilib_arches = multilib_arches if multilib_arches else []`
		`+ self.multilib_method = []`
		`+ for k, v in MULTILIB_METHODS.items():`
		`+ if multilib_method & v:`
		`+ self.multilib_method.append(k)`
		`+`
		`if "boot.iso" in self.results:`
		`self.bootable = True`

server/odcs/server/views.py

file modified

+14 -2

		`@@ -32,7 +32,7 @@`
		`from odcs.server.models import Compose`
		`from odcs.common.types import (`
		`COMPOSE_RESULTS, COMPOSE_FLAGS, COMPOSE_STATES, PUNGI_SOURCE_TYPE_NAMES,`
		`- PungiSourceType)`
		`+ PungiSourceType, MULTILIB_METHODS)`
		`from odcs.server.api_utils import (`
		`pagination_metadata, filter_composes, validate_json_data,`
		`raise_if_input_not_allowed)`
		`@@ -284,6 +284,17 @@`
		`else:`
		`arches = " ".join(conf.arches)`

		`+ multilib_arches = ""`
		`+ if "multilib_arches" in data:`
		`+ multilib_arches = " ".join(data["multilib_arches"])`
		`+`
		`+ multilib_method = MULTILIB_METHODS["none"]`
		`+ if "multilib_method" in data:`
		`+ for name in data["multilib_method"]:`
		`+ if name not in MULTILIB_METHODS:`
		`+ raise ValueError("Unknown multilib method \"%s\"" % name)`
		`+ multilib_method \|= MULTILIB_METHODS[name]`
		`+`
		`raise_if_input_not_allowed(`
		`source_types=source_type, sources=source, results=results,`
		`flags=flags, arches=arches)`
		`@@ -291,7 +302,8 @@`
		`compose = Compose.create(`
		`db.session, self._get_compose_owner(), source_type, source,`
		`results, seconds_to_live,`
		`- packages, flags, sigkeys, arches)`
		`+ packages, flags, sigkeys, arches, multilib_arches=multilib_arches,`
		`+ multilib_method=multilib_method)`
		`db.session.add(compose)`
		# Flush is needed, because we use `before_commit` SQLAlchemy event to
		`# send message and before_commit can be called before flush and`

server/tests/test_backend.py

file modified

+10 -4

		`@@ -316,14 +316,17 @@`
		`attrs["koji_event"] = 123456`
		`attrs["source"] = "123"`
		`attrs["arches"] = "ppc64 x86_64"`
		`+ attrs["multilib_arches"] = "x86_64 i686"`
		`+ attrs["multilib_method"] = 1`
		`for attr, value in attrs.items():`
		`c = Compose.create(`
		`db.session, "me", PungiSourceType.REPO, os.path.join(thisdir, "repo"),`
		`- COMPOSE_RESULTS["repository"], 3600, packages="ed")`
		`+ COMPOSE_RESULTS["repository"], 3600, packages="ed", sigkeys="123")`
		`setattr(c, attr, value)`

		`- # Do not resolve compose for non-existing source...`
		`- if attr != "source":`
		`+ # Do not resolve compose for non-existing source and in case we`
		`+ # change koji_event, because it would be overwriten.`
		`+ if attr not in ["source", "koji_event"]:`
		`resolve_compose(c)`

		`db.session.add(c)`
		`@@ -571,7 +574,8 @@`
		`c = Compose.create(`
		`db.session, "me", PungiSourceType.KOJI_TAG, "f26",`
		`COMPOSE_RESULTS["repository"], 60, packages='pkg1 pkg2 pkg3',`
		`- arches="x86_64 s390")`
		`+ arches="x86_64 s390", multilib_arches="i686 x86_64",`
		`+ multilib_method=1)`
		`c.id = 1`

		`generate_pungi_compose(c)`
		`@@ -587,6 +591,8 @@`
		`self.assertEqual(self.pungi_config.gather_method, "deps")`
		`self.assertEqual(self.pungi_config.pkgset_koji_inherit, True)`
		`self.assertEqual(set(self.pungi_config.arches), set(["x86_64", "s390"]))`
		`+ self.assertEqual(set(self.pungi_config.multilib_arches), set(["i686", "x86_64"]))`
		`+ self.assertEqual(self.pungi_config.multilib_method, ["runtime"])`

		`def test_generate_pungi_compose_nodeps(self):`
		`c = Compose.create(`

server/tests/test_models.py

file modified

+3 -1

		`@@ -66,7 +66,9 @@`
		`'koji_event': None,`
		`'koji_task_id': None,`
		`'packages': None,`
		`- 'arches': 'x86_64'}`
		`+ 'arches': 'x86_64',`
		`+ 'multilib_arches': '',`
		`+ 'multilib_method': 0}`
		`self.assertEqual(c.json(), expected_json)`

server/tests/test_pungi.py

file modified

+18

		`@@ -156,6 +156,24 @@`
		`cfg = self._load_pungi_cfg(template)`
		`self.assertTrue(cfg["pkgset_koji_inherit"])`

		`+ def test_get_pungi_conf_multilib(self):`
		`+ _, mock_path = tempfile.mkstemp()`
		`+ template_path = os.path.abspath(os.path.join(test_dir,`
		`+ "../conf/pungi.conf"))`
		`+ shutil.copy2(template_path, mock_path)`
		`+`
		`+ with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):`
		`+ pungi_cfg = PungiConfig("MBS-512", "1", PungiSourceType.KOJI_TAG,`
		`+ "f26", multilib_arches=["x86_64", "s390x"],`
		`+ multilib_method=3)`
		`+`
		`+ template = pungi_cfg.get_pungi_config()`
		`+ cfg = self._load_pungi_cfg(template)`
		`+ self.assertEqual(set(cfg["multilib"][0][1].keys()), set(["s390x", "x86_64"]))`
		`+ for variant, arch_method_dict in cfg["multilib"]:`
		`+ for method in arch_method_dict.values():`
		`+ self.assertEqual(set(method), set(['runtime', 'devel']))`
		`+`

		`class TestPungi(unittest.TestCase):`

server/tests/test_views.py

file modified

+57 -3

		`@@ -35,7 +35,8 @@`

		`from odcs.server import conf, db, app, login_manager, version`
		`from odcs.server.models import Compose, User`
		`- from odcs.common.types import COMPOSE_STATES, COMPOSE_RESULTS, COMPOSE_FLAGS`
		`+ from odcs.common.types import (COMPOSE_STATES, COMPOSE_RESULTS, COMPOSE_FLAGS,`
		`+ MULTILIB_METHODS)`
		`from odcs.server.pungi import PungiSourceType`
		`from .utils import ModelsBaseTest`
		`from odcs.server.api_utils import validate_json_data`
		`@@ -299,7 +300,9 @@`
		`'koji_event': None,`
		`'koji_task_id': None,`
		`'packages': None,`
		`- 'arches': 'x86_64'}`
		`+ 'arches': 'x86_64',`
		`+ 'multilib_arches': '',`
		`+ 'multilib_method': 0}`
		`self.assertEqual(data, expected_json)`

		`db.session.expire_all()`
		`@@ -429,6 +432,55 @@`
		`c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`self.assertEqual(c.state, COMPOSE_STATES["wait"])`

		`+ def test_submit_build_multilib_arches(self):`
		`+ with self.test_request_context(user='dev'):`
		`+ flask.g.oidc_scopes = [`
		`+ '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')`
		`+ ]`
		`+`
		`+ rv = self.client.post('/api/1/composes/', data=json.dumps(`
		`+ {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']},`
		`+ 'arches': ["ppc64", "s390"], 'multilib_arches': ["x86_64", "ppc64le"]}))`
		`+ data = json.loads(rv.get_data(as_text=True))`
		`+`
		`+ self.assertEqual(data['multilib_arches'], 'x86_64 ppc64le')`
		`+`
		`+ db.session.expire_all()`
		`+ c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`+ self.assertEqual(c.state, COMPOSE_STATES["wait"])`
		`+`
		`+ def test_submit_build_multilib_method(self):`
		`+ with self.test_request_context(user='dev'):`
		`+ flask.g.oidc_scopes = [`
		`+ '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')`
		`+ ]`
		`+`
		`+ rv = self.client.post('/api/1/composes/', data=json.dumps(`
		`+ {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']},`
		`+ 'arches': ["ppc64", "s390"], 'multilib_method': ["runtime", "devel"]}))`
		`+ data = json.loads(rv.get_data(as_text=True))`
		`+`
		`+ self.assertEqual(data['multilib_method'],`
		`+ MULTILIB_METHODS["runtime"] \| MULTILIB_METHODS["devel"])`
		`+`
		`+ db.session.expire_all()`
		`+ c = db.session.query(Compose).filter(Compose.id == 1).one()`
		`+ self.assertEqual(c.state, COMPOSE_STATES["wait"])`
		`+`
		`+ def test_submit_build_multilib_method_unknown(self):`
		`+ with self.test_request_context(user='dev'):`
		`+ flask.g.oidc_scopes = [`
		`+ '{0}{1}'.format(conf.oidc_base_namespace, 'new-compose')`
		`+ ]`
		`+`
		`+ rv = self.client.post('/api/1/composes/', data=json.dumps(`
		`+ {'source': {'type': 'tag', 'source': 'f26', 'packages': ['ed']},`
		`+ 'arches': ["ppc64", "s390"], 'multilib_method': ["foo", "devel"]}))`
		`+ data = json.loads(rv.get_data(as_text=True))`
		`+`
		`+ self.assertEqual(`
		`+ data['message'], 'Unknown multilib method "foo"')`
		`+`
		`def test_submit_build_duplicate_sources(self):`
		`with self.test_request_context(user='dev'):`
		`flask.g.oidc_scopes = [`
		`@@ -898,7 +950,9 @@`
		`'koji_event': None,`
		`'koji_task_id': None,`
		`'packages': None,`
		`- 'arches': 'x86_64'}`
		`+ 'arches': 'x86_64',`
		`+ 'multilib_arches': '',`
		`+ 'multilib_method': 0}`
		`self.assertEqual(data, expected_json)`

		`db.session.expire_all()`

jkaluza commented 5 years ago

This PR allows generating "multilib" composes - these are composes where for example x86_64 repository contains also i686 packages.

There is Pungi multilib configuration which enables that and allows further configuration: https://pagure.io/pungi/blob/master/f/doc/configuration.rst?text=True#_653.

This PR adds new multilib_arches and multilib_method REST API POST fields which can be used to configure the Pungi multilib value. This is why the PR changes views.py and also models.py, because we need to store the new fields in DB.

It also changes the get_reusable_compose code to not reuse composes with different multilib settings.