PR#254: Allow setting lookaside_repos Pungi configuration with extra RPMs to include in a compose. - odcs

odcs

#254 Allow setting lookaside_repos Pungi configuration with extra RPMs to include in a compose.

Merged 5 years ago by jkaluza. Opened 5 years ago by jkaluza.

jkaluza/odcs lookaside-repos into master

Allow setting lookaside_repos Pungi configuration with extra RPMs to include in a compose.

Jan Kaluza • 5 years ago

3de6555

Allow setting the "check_deps" Pungi option using the "check_deps" compose flag.

Jan Kaluza • 5 years ago

1a9933e

README.md

file modified

		`@@ -150,6 +150,7 @@`
		- `all` - All pakages will be considered as multilib.
		- `builds` - List of NVRs defining the Koji builds to include in a compose. Only valid for `tag` and `build` source types. For `tag` source type, the NVRs will be considered
		for inclusion in a compose on top of Koji tag defined by `source`. For `build` source type, only the Koji builds defined by the NVRs will be considered for inclusion. The `packages` still need to be set to include particular packages from the Koji builds in a compose.
		+ - `lookaside_repos` - List of base URLs of RPM repositories which should be considered when choosing packages for a compose.

		The `new_compose` method returns `dict` object describing the compose, for example:

server/conf/pungi.conf

file modified

+10 -1

		`@@ -79,6 +79,16 @@`
		`check_deps = {{ config.check_deps }}`
		`greedy_method = 'build'`

		`+ gather_lookaside_repos = [`
		`+ ('^.*$', {`
		`+ '*': [`
		`+ {%- for lookaside_repo in config.lookaside_repos %}`
		`+ '{{ lookaside_repo }}',`
		`+ {%- endfor %}`
		`+ ]`
		`+ }),`
		`+ ]`
		`+`
		`# CREATEREPO`
		`createrepo_c = True`
		`createrepo_checksum = 'sha256'`
		`@@ -111,4 +121,3 @@`
		`koji_profile = '{{ config.koji_profile }}'`

		`repoclosure_strictness = [('.', {'': 'off'})]`
		`-`

server/odcs/server/backend.py

file modified

+11 -1

		`@@ -463,6 +463,15 @@`
		`old_compose)`
		`continue`

		`+ lookaside_repos = set(compose.lookaside_repos.split(" ")) \`
		`+ if compose.lookaside_repos else set()`
		`+ old_lookaside_repos = set(old_compose.lookaside_repos.split(" ")) \`
		`+ if old_compose.lookaside_repos else set()`
		`+ if lookaside_repos != old_lookaside_repos:`
		`+ log.debug("%r: Cannot reuse %r - lookaside_repos not same", compose,`
		`+ old_compose)`
		`+ continue`
		`+`
		`multilib_arches = set(compose.multilib_arches.split(" ")) \`
		`if compose.multilib_arches else set()`
		`old_multilib_arches = set(old_compose.multilib_arches.split(" ")) \`
		`@@ -642,7 +651,8 @@`
		`arches=compose.arches.split(" "),`
		`multilib_arches=multilib_arches,`
		`multilib_method=compose.multilib_method,`
		`- builds=builds, flags=compose.flags)`
		`+ builds=builds, flags=compose.flags,`
		`+ lookaside_repos=compose.lookaside_repos)`
		`if compose.flags & COMPOSE_FLAGS["no_deps"]:`
		`pungi_cfg.gather_method = "nodeps"`
		`if compose.flags & COMPOSE_FLAGS["no_inheritance"]:`

server/odcs/server/migrations/versions/b2725d046624_.py

file added

+22

		`@@ -0,0 +1,22 @@`
		`+ """add lookaside_repos column`
		`+`
		`+ Revision ID: b2725d046624`
		`+ Revises: 4514febd31fa`
		`+ Create Date: 2019-01-26 09:17:33.392547`
		`+`
		`+ """`
		`+`
		`+ # revision identifiers, used by Alembic.`
		`+ revision = 'b2725d046624'`
		`+ down_revision = '4514febd31fa'`
		`+`
		`+ from alembic import op`
		`+ import sqlalchemy as sa`
		`+`
		`+`
		`+ def upgrade():`
		`+ op.add_column('composes', sa.Column('lookaside_repos', sa.String(), nullable=True))`
		`+`
		`+`
		`+ def downgrade():`
		`+ op.drop_column('composes', 'lookaside_repos')`

server/odcs/server/models.py

file modified

+6 -1

		`@@ -136,12 +136,14 @@`
		`multilib_arches = db.Column(db.String)`
		`# Method to generate multilib compose as defined by python-multilib.`
		`multilib_method = db.Column(db.Integer)`
		`+ # White-space separated lookaside repository URLs.`
		`+ lookaside_repos = db.Column(db.String, nullable=True)`

		`@classmethod`
		`def create(cls, session, owner, source_type, source, results,`
		`seconds_to_live, packages=None, flags=0, sigkeys=None,`
		`arches=None, multilib_arches=None, multilib_method=None,`
		`- builds=None):`
		`+ builds=None, lookaside_repos=None):`
		`now = datetime.utcnow()`
		`compose = cls(`
		`owner=owner,`
		`@@ -158,6 +160,7 @@`
		`multilib_arches=multilib_arches if multilib_arches else "",`
		`multilib_method=multilib_method if multilib_method else 0,`
		`builds=builds,`
		`+ lookaside_repos=lookaside_repos,`
		`)`
		`session.add(compose)`
		`return compose`
		`@@ -189,6 +192,7 @@`
		`multilib_arches=compose.multilib_arches,`
		`multilib_method=compose.multilib_method,`
		`sigkeys=compose.sigkeys,`
		`+ lookaside_repos=compose.lookaside_repos,`
		`)`
		`session.add(compose)`
		`return compose`
		`@@ -316,6 +320,7 @@`
		`'arches': self.arches,`
		`'multilib_arches': self.multilib_arches,`
		`'multilib_method': self.multilib_method,`
		`+ 'lookaside_repos': self.lookaside_repos,`
		`}`

		`@staticmethod`

server/odcs/server/pungi.py

file modified

+2 -1

		`@@ -113,7 +113,7 @@`
		`def __init__(self, release_name, release_version, source_type, source,`
		`packages=None, arches=None, sigkeys=None, results=0,`
		`multilib_arches=None, multilib_method=0, builds=None,`
		`- flags=0):`
		`+ flags=0, lookaside_repos=None):`
		`self.release_name = release_name`
		`self.release_version = release_version`
		`self.bootable = False`
		`@@ -122,6 +122,7 @@`
		`self.source = source`
		`self.koji_profile = conf.koji_profile`
		`self.pkgset_koji_inherit = True`
		`+ self.lookaside_repos = lookaside_repos.split(" ") if lookaside_repos else []`
		`if arches:`
		`self.arches = arches`
		`else:`

server/odcs/server/views.py

file modified

+6 -1

		`@@ -297,6 +297,10 @@`
		`if "multilib_arches" in data:`
		`multilib_arches = " ".join(data["multilib_arches"])`

		`+ lookaside_repos = ""`
		`+ if "lookaside_repos" in data:`
		`+ lookaside_repos = " ".join(data["lookaside_repos"])`
		`+`
		`multilib_method = MULTILIB_METHODS["none"]`
		`if "multilib_method" in data:`
		`for name in data["multilib_method"]:`
		`@@ -312,7 +316,8 @@`
		`db.session, self._get_compose_owner(), source_type, source,`
		`results, seconds_to_live,`
		`packages, flags, sigkeys, arches, multilib_arches=multilib_arches,`
		`- multilib_method=multilib_method, builds=builds)`
		`+ multilib_method=multilib_method, builds=builds,`
		`+ lookaside_repos=lookaside_repos)`
		`db.session.add(compose)`
		# Flush is needed, because we use `before_commit` SQLAlchemy event to
		`# send message and before_commit can be called before flush and`

server/tests/test_backend.py

file modified

		`@@ -389,6 +389,7 @@`
		`attrs["arches"] = "ppc64 x86_64"`
		`attrs["multilib_arches"] = "x86_64 i686"`
		`attrs["multilib_method"] = 1`
		`+ attrs["lookaside_repos"] = "foo bar"`
		`for attr, value in attrs.items():`
		`c = Compose.create(`
		`db.session, "me", PungiSourceType.REPO, os.path.join(thisdir, "repo"),`

server/tests/test_models.py

file modified

+2 -1

		`@@ -68,7 +68,8 @@`
		`'builds': None,`
		`'arches': 'x86_64',`
		`'multilib_arches': '',`
		`- 'multilib_method': 0}`
		`+ 'multilib_method': 0,`
		`+ 'lookaside_repos': None}`
		`self.assertEqual(c.json(), expected_json)`

		`def test_create_copy(self):`

server/tests/test_pungi.py

file modified

+17

		`@@ -261,6 +261,23 @@`
		`self.assertEqual(cfg["pkgset_koji_tag"], 'f26')`
		`self.assertTrue("additional_packages" not in cfg)`

		`+ def test_get_pungi_conf_lookaside_repos(self):`
		`+ _, mock_path = tempfile.mkstemp()`
		`+ template_path = os.path.abspath(os.path.join(test_dir,`
		`+ "../conf/pungi.conf"))`
		`+ shutil.copy2(template_path, mock_path)`
		`+`
		`+ with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):`
		`+ pungi_cfg = PungiConfig(`
		`+ "MBS-512", "1", PungiSourceType.KOJI_TAG, "f26",`
		`+ lookaside_repos="foo bar")`
		`+`
		`+ template = pungi_cfg.get_pungi_config()`
		`+ cfg = self._load_pungi_cfg(template)`
		`+ self.assertEqual(`
		`+ cfg["gather_lookaside_repos"],`
		`+ [(u'^.$', {u'': [u'foo', u'bar']})])`
		`+`

		`class TestPungi(unittest.TestCase):`

server/tests/test_views.py

file modified

+4 -2

		`@@ -304,7 +304,8 @@`
		`'builds': None,`
		`'arches': 'x86_64',`
		`'multilib_arches': '',`
		`- 'multilib_method': 0}`
		`+ 'multilib_method': 0,`
		`+ 'lookaside_repos': ''}`
		`self.assertEqual(data, expected_json)`

		`db.session.expire_all()`
		`@@ -1009,7 +1010,8 @@`
		`'builds': None,`
		`'arches': 'x86_64',`
		`'multilib_arches': '',`
		`- 'multilib_method': 0}`
		`+ 'multilib_method': 0,`
		`+ 'lookaside_repos': ''}`
		`self.assertEqual(data, expected_json)`

		`db.session.expire_all()`

jkaluza commented 5 years ago

This is useful for our RCM friends who want to be able to generate compose with extra RPMs
on top of existing compose referenced by lookaside repository.

Depends on PR #253.

jkaluza commented 5 years ago

@lsedlar ^

lsedlar commented 5 years ago

I tested a few things on Pungi level:

$basearch works as expected, which is nice and can simplify using this feature
specifying repos with incompatible arch should also not be a problem (technically noarch packages could be in multiple places, but that is not causing any problems).

So :thumbsup: from me.

Using many lookasides can slow the compose, since it will need to download the repodata, which can be fairly big.

And a minor detail: I'm not RCM :sweat_smile:

rebased onto 1a9933e

5 years ago

jkaluza commented 5 years ago

Rebased against latest master.

Pull-Request has been merged by jkaluza

5 years ago