#229 Reuse last koji event
Merged 7 months ago by jkaluza. Opened 7 months ago by cqi.

file modified
+43 -10

@@ -40,6 +40,15 @@ 

  import odcs.server.mbs

  import defusedxml.ElementTree

  

+ # Cache last event for each tag since that a compose was generated from that

+ # tag.

+ # This is a mapping from tag name to koji event id. For example,

+ # {

+ #     'tag1': 123456,

+ #     'tag2': 123476,

+ # }

+ LAST_EVENTS_CACHE = {}

+ 

  

  class BackendThread(object):

      """

@@ -269,6 +278,21 @@ 

      return ids

  

  

+ def tag_changed(koji_session, tag, koji_event):

+     """

+     Check if tag and its parents in the inheritance have changed since a

+     particular koji event

+ 

+     :param koji_session: instance of :class:`ClientSession`.

+     :param str tag: tag name.

+     :param int koji_event: the koji event id.

+     :return: True if changed, otherwise False.

+     :rtype: bool

+     """

+     tags = koji_get_inherited_tags(koji_session, tag)

+     return koji_session.tagChangedSinceEvent(koji_event, tags)

+ 

+ 

  def resolve_compose(compose):

      """

      Resolves various general compose values to the real ones. For example:

@@ -285,13 +309,28 @@ 

          revision = e.find("{http://linux.duke.edu/metadata/repo}revision").text

          compose.koji_event = int(revision)

      elif compose.source_type == PungiSourceType.KOJI_TAG:

+         global LAST_EVENTS_CACHE

+         koji_session = create_koji_session()

          # If compose.koji_event is set, it means that we are regenerating

          # previous compose and we have to respect the previous koji_event to

          # get the same results.

          if not compose.koji_event:

-             koji_session = create_koji_session()

-             compose.koji_event = int(koji_session.getLastEvent()['id'])

+             if compose.source not in LAST_EVENTS_CACHE:

+                 event_id = int(koji_session.getLastEvent()['id'])

+             elif tag_changed(koji_session,

+                              compose.source,

+                              LAST_EVENTS_CACHE[compose.source]):

+                 event_id = int(koji_session.getLastEvent()['id'])

+             else:

+                 event_id = LAST_EVENTS_CACHE[compose.source]

+                 log.info('Reuse koji event %s to generate compose %s from source %s',

+                          event_id, compose.id, compose.source)

+             compose.koji_event = event_id

+             # event_id could be a new koji event ID. Cache it for next potential

+             # reuse for same tag.

+             LAST_EVENTS_CACHE[compose.source] = event_id

      elif compose.source_type == PungiSourceType.MODULE:

+ 

          # Resolve the latest release of modules which do not have the release

          # string defined in the compose.source.

          mbs = odcs.server.mbs.MBS(conf)

@@ -396,15 +435,9 @@ 

              # For KOJI_TAG compose, check that all the inherited tags by our

              # Koji tag have not changed since previous old_compose.

              koji_session = create_koji_session()

-             tags = koji_get_inherited_tags(koji_session, compose.source)

-             if not tags:

-                 continue

-             changed = koji_session.tagChangedSinceEvent(

-                 old_compose.koji_event, tags)

-             if changed:

+             if tag_changed(koji_session, compose.source, old_compose.koji_event):

                  log.debug("%r: Cannot reuse %r - one of the tags changed "

-                           "since previous compose: %r", compose, old_compose,

-                           tags)

+                           "since previous compose.", compose, old_compose)

                  continue

          elif compose.koji_event != old_compose.koji_event:

              log.debug("%r: Cannot reuse %r - koji_events not same, %d != %d",

@@ -516,6 +516,63 @@ 

              r'Error while generating compose: Expected exception\n'

              'Compose failed for unknown reason*')

  

+     @patch('odcs.server.backend.tag_changed', return_value=True)

+     @patch('odcs.server.backend.create_koji_session')

+     def test_resolve_compose_from_koji_tag_get_last_event_if_tag_changed(

+             self, create_koji_session, tag_changed):

+         session = create_koji_session.return_value

+         fake_koji_event = {'id': 234567}

+         session.getLastEvent.return_value = fake_koji_event

+ 

+         c = Compose.create(

+             db.session, "me", PungiSourceType.KOJI_TAG, "foo-1",

+             COMPOSE_RESULTS["repository"], 3600)

+         db.session.add(c)

+         db.session.commit()

+ 

+         with patch.dict('odcs.server.backend.LAST_EVENTS_CACHE', {'foo-1': 123456}):

+             resolve_compose(c)

+             c.koji_event = fake_koji_event['id']

+ 

+     @patch('odcs.server.backend.tag_changed')

+     @patch('odcs.server.backend.create_koji_session')

+     def test_resolve_compose_from_koji_tag_reuse_koji_event_if_tag_not_changed(

+             self, create_koji_session, tag_changed):

+         tag_changed.return_value = False

+         session = create_koji_session.return_value

+ 

+         c = Compose.create(

+             db.session, "me", PungiSourceType.KOJI_TAG, "foo-1",

+             COMPOSE_RESULTS["repository"], 3600)

+         db.session.add(c)

+         db.session.commit()

+ 

+         with patch.dict('odcs.server.backend.LAST_EVENTS_CACHE', {'foo-1': 123456}):

+             resolve_compose(c)

+             c.koji_event = 123456

+ 

+             session.getLastEvent.assert_not_called()

+ 

+     @patch('odcs.server.backend.tag_changed')

+     @patch('odcs.server.backend.create_koji_session')

+     def test_resolve_compose_from_koji_tag_get_last_koji_event_if_tag_not_cached(

+             self, create_koji_session, tag_changed):

+         fake_koji_event = {'id': 789065}

+         session = create_koji_session.return_value

+         session.getLastEvent.return_value = fake_koji_event

+ 

+         c = Compose.create(

+             db.session, "me", PungiSourceType.KOJI_TAG, "foo-1",

+             COMPOSE_RESULTS["repository"], 3600)

+         db.session.add(c)

+         db.session.commit()

+ 

+         with patch.dict('odcs.server.backend.LAST_EVENTS_CACHE', {'bar-2': 123456}):

+             resolve_compose(c)

+             c.koji_event = fake_koji_event['id']

+ 

+             tag_changed.assert_not_called()

+ 

  

  class TestGeneratePungiCompose(ModelsBaseTest):

  

While resolving a compose before generating it from a particular koji
tag, if tag and its parents through out its inheritance have not changed
since last koji event, reuse that koji event again instead of call API
to get a new one.

A module dict is used as a simple cache during ODCS execution. It maps
koji event ID from a koji tag.

Signed-off-by: Chenxiong Qi cqi@redhat.com

rebased onto e25b9a31434ad5158cc3d4860d0b0070bdf29969

7 months ago

Can you log.info("%r: Reusing cached koji_event.", compose) here?

rebased onto 0eed98a

7 months ago

Pull-Request has been merged by jkaluza

7 months ago