#254 Allow setting lookaside_repos Pungi configuration with extra RPMs to include in a compose.
Merged 5 months ago by jkaluza. Opened 5 months ago by jkaluza.
jkaluza/odcs lookaside-repos  into  master

file modified
+1

@@ -150,6 +150,7 @@ 

      - `all` - All pakages will be considered as multilib.

  - `builds` - List of NVRs defining the Koji builds to include in a compose. Only valid for `tag` and `build` source types. For `tag` source type, the NVRs will be considered

  for inclusion in a compose on top of Koji tag defined by `source`. For `build` source type, only the Koji builds defined by the NVRs will be considered for inclusion. The `packages` still need to be set to include particular packages from the Koji builds in a compose.

+ - `lookaside_repos` - List of base URLs of RPM repositories which should be considered when choosing packages for a compose.

  

  The `new_compose` method returns `dict` object describing the compose, for example:

  

file modified
+10 -1

@@ -79,6 +79,16 @@ 

  check_deps = {{ config.check_deps }}

  greedy_method = 'build'

  

+ gather_lookaside_repos = [

+     ('^.*$', {

+         '*': [

+ {%- for lookaside_repo in config.lookaside_repos %}

+             '{{ lookaside_repo }}',

+ {%- endfor %}

+         ]

+     }),

+ ]

+ 

  # CREATEREPO

  createrepo_c = True

  createrepo_checksum = 'sha256'

@@ -111,4 +121,3 @@ 

  koji_profile = '{{ config.koji_profile }}'

  

  repoclosure_strictness = [('.*', {'*': 'off'})]

- 

file modified
+11 -1

@@ -463,6 +463,15 @@ 

                        old_compose)

              continue

  

+         lookaside_repos = set(compose.lookaside_repos.split(" ")) \

+             if compose.lookaside_repos else set()

+         old_lookaside_repos = set(old_compose.lookaside_repos.split(" ")) \

+             if old_compose.lookaside_repos else set()

+         if lookaside_repos != old_lookaside_repos:

+             log.debug("%r: Cannot reuse %r - lookaside_repos not same", compose,

+                       old_compose)

+             continue

+ 

          multilib_arches = set(compose.multilib_arches.split(" ")) \

              if compose.multilib_arches else set()

          old_multilib_arches = set(old_compose.multilib_arches.split(" ")) \

@@ -642,7 +651,8 @@ 

                                      arches=compose.arches.split(" "),

                                      multilib_arches=multilib_arches,

                                      multilib_method=compose.multilib_method,

-                                     builds=builds, flags=compose.flags)

+                                     builds=builds, flags=compose.flags,

+                                     lookaside_repos=compose.lookaside_repos)

              if compose.flags & COMPOSE_FLAGS["no_deps"]:

                  pungi_cfg.gather_method = "nodeps"

              if compose.flags & COMPOSE_FLAGS["no_inheritance"]:

@@ -0,0 +1,22 @@ 

+ """add lookaside_repos column

+ 

+ Revision ID: b2725d046624

+ Revises: 4514febd31fa

+ Create Date: 2019-01-26 09:17:33.392547

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = 'b2725d046624'

+ down_revision = '4514febd31fa'

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     op.add_column('composes', sa.Column('lookaside_repos', sa.String(), nullable=True))

+ 

+ 

+ def downgrade():

+     op.drop_column('composes', 'lookaside_repos')

file modified
+6 -1

@@ -136,12 +136,14 @@ 

      multilib_arches = db.Column(db.String)

      # Method to generate multilib compose as defined by python-multilib.

      multilib_method = db.Column(db.Integer)

+     # White-space separated lookaside repository URLs.

+     lookaside_repos = db.Column(db.String, nullable=True)

  

      @classmethod

      def create(cls, session, owner, source_type, source, results,

                 seconds_to_live, packages=None, flags=0, sigkeys=None,

                 arches=None, multilib_arches=None, multilib_method=None,

-                builds=None):

+                builds=None, lookaside_repos=None):

          now = datetime.utcnow()

          compose = cls(

              owner=owner,

@@ -158,6 +160,7 @@ 

              multilib_arches=multilib_arches if multilib_arches else "",

              multilib_method=multilib_method if multilib_method else 0,

              builds=builds,

+             lookaside_repos=lookaside_repos,

          )

          session.add(compose)

          return compose

@@ -189,6 +192,7 @@ 

              multilib_arches=compose.multilib_arches,

              multilib_method=compose.multilib_method,

              sigkeys=compose.sigkeys,

+             lookaside_repos=compose.lookaside_repos,

          )

          session.add(compose)

          return compose

@@ -316,6 +320,7 @@ 

              'arches': self.arches,

              'multilib_arches': self.multilib_arches,

              'multilib_method': self.multilib_method,

+             'lookaside_repos': self.lookaside_repos,

          }

  

      @staticmethod

file modified
+2 -1

@@ -113,7 +113,7 @@ 

      def __init__(self, release_name, release_version, source_type, source,

                   packages=None, arches=None, sigkeys=None, results=0,

                   multilib_arches=None, multilib_method=0, builds=None,

-                  flags=0):

+                  flags=0, lookaside_repos=None):

          self.release_name = release_name

          self.release_version = release_version

          self.bootable = False

@@ -122,6 +122,7 @@ 

          self.source = source

          self.koji_profile = conf.koji_profile

          self.pkgset_koji_inherit = True

+         self.lookaside_repos = lookaside_repos.split(" ") if lookaside_repos else []

          if arches:

              self.arches = arches

          else:

file modified
+6 -1

@@ -297,6 +297,10 @@ 

          if "multilib_arches" in data:

              multilib_arches = " ".join(data["multilib_arches"])

  

+         lookaside_repos = ""

+         if "lookaside_repos" in data:

+             lookaside_repos = " ".join(data["lookaside_repos"])

+ 

          multilib_method = MULTILIB_METHODS["none"]

          if "multilib_method" in data:

              for name in data["multilib_method"]:

@@ -312,7 +316,8 @@ 

              db.session, self._get_compose_owner(), source_type, source,

              results, seconds_to_live,

              packages, flags, sigkeys, arches, multilib_arches=multilib_arches,

-             multilib_method=multilib_method, builds=builds)

+             multilib_method=multilib_method, builds=builds,

+             lookaside_repos=lookaside_repos)

          db.session.add(compose)

          # Flush is needed, because we use `before_commit` SQLAlchemy event to

          # send message and before_commit can be called before flush and

@@ -389,6 +389,7 @@ 

          attrs["arches"] = "ppc64 x86_64"

          attrs["multilib_arches"] = "x86_64 i686"

          attrs["multilib_method"] = 1

+         attrs["lookaside_repos"] = "foo bar"

          for attr, value in attrs.items():

              c = Compose.create(

                  db.session, "me", PungiSourceType.REPO, os.path.join(thisdir, "repo"),

file modified
+2 -1

@@ -68,7 +68,8 @@ 

                           'builds': None,

                           'arches': 'x86_64',

                           'multilib_arches': '',

-                          'multilib_method': 0}

+                          'multilib_method': 0,

+                          'lookaside_repos': None}

          self.assertEqual(c.json(), expected_json)

  

      def test_create_copy(self):

@@ -261,6 +261,23 @@ 

              self.assertEqual(cfg["pkgset_koji_tag"], 'f26')

              self.assertTrue("additional_packages" not in cfg)

  

+     def test_get_pungi_conf_lookaside_repos(self):

+         _, mock_path = tempfile.mkstemp()

+         template_path = os.path.abspath(os.path.join(test_dir,

+                                                      "../conf/pungi.conf"))

+         shutil.copy2(template_path, mock_path)

+ 

+         with patch("odcs.server.pungi.conf.pungi_conf_path", mock_path):

+             pungi_cfg = PungiConfig(

+                 "MBS-512", "1", PungiSourceType.KOJI_TAG, "f26",

+                 lookaside_repos="foo bar")

+ 

+             template = pungi_cfg.get_pungi_config()

+             cfg = self._load_pungi_cfg(template)

+             self.assertEqual(

+                 cfg["gather_lookaside_repos"],

+                 [(u'^.*$', {u'*': [u'foo', u'bar']})])

+ 

  

  class TestPungi(unittest.TestCase):

  

file modified
+4 -2

@@ -304,7 +304,8 @@ 

                           'builds': None,

                           'arches': 'x86_64',

                           'multilib_arches': '',

-                          'multilib_method': 0}

+                          'multilib_method': 0,

+                          'lookaside_repos': ''}

          self.assertEqual(data, expected_json)

  

          db.session.expire_all()

@@ -1009,7 +1010,8 @@ 

                           'builds': None,

                           'arches': 'x86_64',

                           'multilib_arches': '',

-                          'multilib_method': 0}

+                          'multilib_method': 0,

+                          'lookaside_repos': ''}

          self.assertEqual(data, expected_json)

  

          db.session.expire_all()

This is useful for our RCM friends who want to be able to generate compose with extra RPMs
on top of existing compose referenced by lookaside repository.

Depends on PR #253.

I tested a few things on Pungi level:

  • $basearch works as expected, which is nice and can simplify using this feature
  • specifying repos with incompatible arch should also not be a problem (technically noarch packages could be in multiple places, but that is not causing any problems).

So :thumbsup: from me.

Using many lookasides can slow the compose, since it will need to download the repodata, which can be fairly big.

And a minor detail: I'm not RCM :sweat_smile:

rebased onto 1a9933e

5 months ago

Rebased against latest master.

Pull-Request has been merged by jkaluza

5 months ago