#521 Populate some fields for manual rebuilds
Merged 4 years ago by apaplaus. Opened 4 years ago by apaplaus.
apaplaus/freshmaker requester_field_population  into  master

file modified
+12 -2
@@ -314,7 +314,8 @@ 

      """

  

      def __init__(self, msg_id, advisory, container_images,

-                  requester_metadata_json=None, freshmaker_event_id=None, **kwargs):

+                  requester_metadata_json=None, freshmaker_event_id=None,

+                  requester=None, **kwargs):

          """

          Creates new ManualRebuildWithAdvisoryEvent.

  
@@ -322,14 +323,17 @@ 

          :param ErrataAdvisory advisory: Errata advisory associated with event.

          :param list container_images: List of NVRs of images to rebuild or

              empty list to rebuild all images affected by the advisory.

+         :param requester_metadata_json: JSON of additional information about rebuild

          :param freshmaker_event_id: Freshmaker event id on which this manual rebuild

              is based off on.

+         :param requester: name of requester of rebuild

          """

          super(ManualRebuildWithAdvisoryEvent, self).__init__(

              msg_id, advisory, **kwargs)

          self.container_images = container_images

          self.requester_metadata_json = requester_metadata_json

          self.freshmaker_event_id = freshmaker_event_id

+         self.requester = requester

  

  

  class BrewSignRPMEvent(BaseEvent):
@@ -416,7 +420,8 @@ 

      """Event triggered via API endpoint /async-builds"""

  

      def __init__(self, msg_id, dist_git_branch, container_images,

-                  freshmaker_event_id=None, brew_target=None, dry_run=False):

+                  freshmaker_event_id=None, brew_target=None, dry_run=False,

+                  requester=None, requester_metadata_json=None):

          """Initialize this event

  

          :param str msg_id: the message id.
@@ -434,6 +439,9 @@ 

          :param brew_target: the Brew target for the build. If not set, the

              previous ``buildContainer`` task build target will be used.

          :type brew_target: str or None

+         :param dry_run: True if the event should be handled in DRY_RUN mode.

+         :param requester: name of requester of rebuild

+         :param requester_metadata_json: JSON of additional information about rebuild

          """

          super(FreshmakerAsyncManualBuildEvent, self).__init__(

              msg_id, manual=True, dry_run=dry_run)
@@ -441,3 +449,5 @@ 

          self.container_images = container_images

          self.freshmaker_event_id = freshmaker_event_id

          self.brew_target = brew_target

+         self.requester = requester

+         self.requester_metadata_json = requester_metadata_json

@@ -27,8 +27,8 @@ 

  from freshmaker.models import Event

  from freshmaker.errata import Errata

  from freshmaker.pulp import Pulp

- from freshmaker.events import (ErrataAdvisoryStateChangedEvent,

-                                ManualRebuildWithAdvisoryEvent)

+ from freshmaker.events import (

+     ErrataAdvisoryStateChangedEvent, ManualRebuildWithAdvisoryEvent)

  from freshmaker.handlers import ContainerBuildHandler, fail_event_on_handler_exception

  from freshmaker.types import EventState, ArtifactType, ArtifactBuildState

  
@@ -53,6 +53,7 @@ 

              self.force_dry_run()

  

          db_event = Event.get_or_create_from_event(db.session, event)

+ 

          self.set_context(db_event)

  

          # Check if we are allowed to build this advisory.

@@ -27,8 +27,8 @@ 

  import kobo

  

  from freshmaker import conf, db

- from freshmaker.events import ErrataAdvisoryRPMsSignedEvent

- from freshmaker.events import ManualRebuildWithAdvisoryEvent

+ from freshmaker.events import (

+     ErrataAdvisoryRPMsSignedEvent, ManualRebuildWithAdvisoryEvent)

  from freshmaker.handlers import ContainerBuildHandler, fail_event_on_handler_exception

  from freshmaker.lightblue import LightBlue

  from freshmaker.pulp import Pulp
@@ -72,9 +72,8 @@ 

          # Generate the Database representation of `event`, it can be

          # triggered by user, we want to track what happened

  

-         db_event = Event.get_or_create(

-             db.session, event.msg_id, event.search_key, event.__class__,

-             released=False, manual=event.manual)

+         db_event = Event.get_or_create_from_event(db.session, event)

+ 

          db.session.commit()

          self.set_context(db_event)

  
@@ -210,8 +209,7 @@ 

              stored into database.

          :rtype: dict

          """

-         db_event = Event.get_or_create(

-             db.session, event.msg_id, event.search_key, event.__class__)

+         db_event = Event.get_or_create_from_event(db.session, event)

  

          # Used as tmp dict with {brew_build_nvr: ArtifactBuild, ...} mapping.

          builds = builds or {}

file modified
+32 -4
@@ -172,7 +172,8 @@ 

  

      @classmethod

      def create(cls, session, message_id, search_key, event_type, released=True,

-                state=None, manual=False, dry_run=False, requester=None):

+                state=None, manual=False, dry_run=False, requester=None,

+                requested_rebuilds=None, requester_metadata=None):

          if event_type in EVENT_TYPES:

              event_type = EVENT_TYPES[event_type]

          now = datetime.utcnow()
@@ -186,6 +187,8 @@ 

              manual_triggered=manual,

              dry_run=dry_run,

              requester=requester,

+             requested_rebuilds=requested_rebuilds,

+             requester_metadata=requester_metadata,

          )

          session.add(event)

          return event
@@ -206,22 +209,47 @@ 

  

      @classmethod

      def get_or_create(cls, session, message_id, search_key, event_type,

-                       released=True, manual=False, dry_run=False):

+                       released=True, manual=False, dry_run=False,

+                       requester=None, requested_rebuilds=None,

+                       requester_metadata=None):

          instance = cls.get(session, message_id)

          if instance:

              return instance

          instance = cls.create(

              session, message_id, search_key, event_type,

-             released=released, manual=manual, dry_run=dry_run)

+             released=released, manual=manual, dry_run=dry_run,

+             requester=requester, requested_rebuilds=requested_rebuilds,

+             requester_metadata=requester_metadata)

          session.commit()

          return instance

  

      @classmethod

      def get_or_create_from_event(cls, session, event, released=True):

+         # we must extract all needed arguments,

+         # because event might not have some of them so we will use defaults

+         requester = getattr(event, "requester", None)

+         requested_rebuilds_list = getattr(event, "container_images", None)

+         requested_rebuilds = None

+         # make sure 'container_images' field is a list and convert it to str

+         if requested_rebuilds_list is not None and \

+                 isinstance(requested_rebuilds_list, list):

+             requested_rebuilds = " ".join(requested_rebuilds_list)

+         requester_metadata = getattr(event, "requester_metadata_json", None)

+         if requester_metadata is not None:

+             # try to convert JSON into str, if it's invalid use None

+             try:

+                 requester_metadata = json.dumps(requester_metadata)

+             except TypeError:

+                 log.warning("requester_metadata_json field is ill-formatted: %s",

+                             requester_metadata)

+                 requester_metadata = None

+ 

          return cls.get_or_create(session, event.msg_id,

                                   event.search_key, event.__class__,

                                   released=released, manual=event.manual,

-                                  dry_run=event.dry_run)

+                                  dry_run=event.dry_run, requester=requester,

+                                  requested_rebuilds=requested_rebuilds,

+                                  requester_metadata=requester_metadata)

  

      @classmethod

      def get_unreleased(cls, session, states=None):

@@ -48,7 +48,9 @@ 

              msg_id, data.get('dist_git_branch'), data.get('container_images', []),

              freshmaker_event_id=data.get('freshmaker_event_id'),

              brew_target=data.get('brew_target'),

-             dry_run=data.get('dry_run', False))

+             dry_run=data.get('dry_run', False),

+             requester=data.get('requester', None),

+             requester_metadata_json=data.get("metadata", None))

  

          return event

  

@@ -52,7 +52,8 @@ 

  

          event = ManualRebuildWithAdvisoryEvent(

              msg_id, advisory, data.get("container_images", []), data.get("metadata", None),

-             freshmaker_event_id=data.get('freshmaker_event_id'), manual=True, dry_run=dry_run)

+             freshmaker_event_id=data.get('freshmaker_event_id'), manual=True,

+             dry_run=dry_run, requester=data.get('requester', None))

  

          return event

  

file modified
+8
@@ -525,6 +525,10 @@ 

          # added to DB) to backend using UMB messaging. Backend will then

          # re-generate the event and start handling it.

          data["msg_id"] = db_event.message_id

+ 

+         # add information about requester

+         data["requester"] = db_event.requester

+ 

          messaging.publish("manual.rebuild", data)

  

          # Return back the JSON representation of Event to client.
@@ -611,6 +615,10 @@ 

          # added to DB) to backend using UMB messaging. Backend will then

          # re-generate the event and start handling it.

          data["msg_id"] = db_event.message_id

+ 

+         # add information about requester

+         data["requester"] = db_event.requester

+ 

          messaging.publish("async.manual.build", data)

  

          # Return back the JSON representation of Event to client.

@@ -28,7 +28,8 @@ 

  from freshmaker import db, events

  from freshmaker.events import (

      ErrataAdvisoryRPMsSignedEvent,

-     ManualRebuildWithAdvisoryEvent)

+     ManualRebuildWithAdvisoryEvent,

+     BaseEvent)

  from freshmaker.handlers.koji import RebuildImagesOnRPMAdvisoryChange

  from freshmaker.lightblue import ContainerImage

  from freshmaker.models import Event, Compose, ArtifactBuild, EVENT_TYPES
@@ -249,6 +250,23 @@ 

              ret = handler.can_handle(event)

              self.assertFalse(ret)

  

+     @patch.object(freshmaker.conf, 'dry_run', new=True)

+     def test_requester_on_manual_rebuild(self):

+         event = ManualRebuildWithAdvisoryEvent(

+             "123",

+             ErrataAdvisory(123, "RHBA-2017", "REL_PREP", ["rpm"],

+                            security_impact="",

+                            product_short_name="product"),

+             ["foo-container", "bar-container"],

+             requester="requester1")

+         handler = RebuildImagesOnRPMAdvisoryChange()

+         ret = handler.can_handle(event)

+         self.assertTrue(ret)

+         handler.handle(event)

+ 

+         db_event = Event.get(db.session, message_id='123')

+         self.assertEqual(db_event.requester, 'requester1')

+ 

      @patch.object(freshmaker.conf, 'handler_build_whitelist', new={

          'RebuildImagesOnRPMAdvisoryChange': {

              'image': {'product_short_name': 'foo'}
@@ -998,7 +1016,8 @@ 

      def setUp(self):

          super(TestRecordBatchesImages, self).setUp()

  

-         self.mock_event = Mock(msg_id='msg-id', search_key=12345)

+         self.mock_event = Mock(spec=BaseEvent, msg_id='msg-id', search_key=12345,

+                                manual=False, dry_run=False)

  

          self.patcher = helpers.Patcher(

              'freshmaker.handlers.koji.RebuildImagesOnRPMAdvisoryChange.')

file modified
+34 -6
@@ -677,7 +677,8 @@ 

              u'requester_metadata': {}})

          publish.assert_called_once_with(

              'manual.rebuild',

-             {'msg_id': 'manual_rebuild_123', u'errata_id': 1})

+             {'msg_id': 'manual_rebuild_123', u'errata_id': 1,

+              'requester': 'root'})

  

      @patch('freshmaker.messaging.publish')

      @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.'
@@ -697,7 +698,8 @@ 

          self.assertEqual(data['dry_run'], True)

          publish.assert_called_once_with(

              'manual.rebuild',

-             {'msg_id': 'manual_rebuild_123', u'errata_id': 1, 'dry_run': True})

+             {'msg_id': 'manual_rebuild_123', u'errata_id': 1, 'dry_run': True,

+              'requester': 'root'})

  

      @patch('freshmaker.messaging.publish')

      @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.'
@@ -721,7 +723,7 @@ 

          publish.assert_called_once_with(

              'manual.rebuild',

              {'msg_id': 'manual_rebuild_123', u'errata_id': 1,

-              'container_images': ["foo-1-1", "bar-1-1"]})

+              'container_images': ["foo-1-1", "bar-1-1"], 'requester': 'root'})

  

      @patch('freshmaker.messaging.publish')

      @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.'
@@ -745,7 +747,30 @@ 

          publish.assert_called_once_with(

              'manual.rebuild',

              {'msg_id': 'manual_rebuild_123', u'errata_id': 1,

-              'metadata': {"foo": ["bar"]}})

+              'metadata': {"foo": ["bar"]}, 'requester': 'root'})

+ 

+     @patch('freshmaker.messaging.publish')

+     @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.'

+            'from_advisory_id')

+     @patch('freshmaker.parsers.internal.manual_rebuild.time.time')

+     def test_manual_rebuild_requester(self, time, from_advisory_id, publish):

+         time.return_value = 123

+         from_advisory_id.return_value = ErrataAdvisory(

+             123, 'name', 'REL_PREP', ['rpm'])

+ 

+         payload = {

+             'errata_id': 1,

+         }

+         with self.test_request_context(user='root'):

+             resp = self.client.post('/api/1/builds/', json=payload, content_type='application/json')

+         data = resp.json

+ 

+         # Other fields are predictible.

+         self.assertEqual(data['requester'], "root")

+         publish.assert_called_once_with(

+             'manual.rebuild',

+             {'msg_id': 'manual_rebuild_123', u'errata_id': 1,

+              'requester': 'root'})

  

      @patch('freshmaker.messaging.publish')

      @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.'
@@ -778,7 +803,8 @@ 

          publish.assert_called_once_with(

              'manual.rebuild',

              {'msg_id': 'manual_rebuild_123', u'errata_id': 103,

-              'container_images': ["foo-1-1"], 'freshmaker_event_id': 1})

+              'container_images': ["foo-1-1"], 'freshmaker_event_id': 1,

+              'requester': 'root'})

  

      @patch('freshmaker.messaging.publish')

      @patch('freshmaker.parsers.internal.manual_rebuild.ErrataAdvisory.'
@@ -952,7 +978,8 @@ 

              {

                  'msg_id': 'async_build_123',

                  'dist_git_branch': 'master',

-                 'container_images': ['foo-1-1', 'bar-1-1']

+                 'container_images': ['foo-1-1', 'bar-1-1'],

+                 'requester': 'root',

              })

  

      @patch('freshmaker.messaging.publish')
@@ -979,6 +1006,7 @@ 

                  'dist_git_branch': 'master',

                  'container_images': ['foo-1-1', 'bar-1-1'],

                  'dry_run': True,

+                 'requester': 'root',

              })

  

      def test_async_build_with_non_async_event(self):

Populate requester, requester_metadata and requested_rebuilds fields
when consumers create new events in situation when they can't find them
in db.

RESOLVE: CLOUDWF-507

Flake8 complain about this line: E265 block comment should start with '# ' -- you are missing a space after the #

Flake8 complain for: E231 missing whitespace after ',', you need to add a space before requester.

1 new commit added

  • Code style fixes
4 years ago

I'm not pretty clear about in which situation the requester, requester_metadata and requested_rebuilds can be missed. It would be better to explain that in commit message.

But if that happens when the event is not found in DB, how about updating Event.get_or_create_from_event and Event.get_or_create to have these data recorded?

I see, so this is caused by the message published doesn't include requester, right? Could you also fix this in AsyncBuildAPI.post?

I see, so this is caused by the message published doesn't include requester, right? Could you also fix this in AsyncBuildAPI.post?

Yes, and when consumer can't find event in db he creates new one with info from message, and if message doesn't have requester, requester_metadata or requested_rebuilds, missing fields will be empty in new event.

I will fix it for AsyncBuild too.

I'm not pretty clear about in which situation the requester, requester_metadata and requested_rebuilds can be missed. It would be better to explain that in commit message.
But if that happens when the event is not found in DB, how about updating Event.get_or_create_from_event and Event.get_or_create to have these data recorded?

get_or_create can't be used because it doesn't have access to requester information, because if doesn't get Event object. And get_or_create_from_event could do populating of this info, but we are using get_or_create instead in places where processing new events. And those two functions do almost the same, but still they where created for something so I would let them be where they are now .

get_or_create can't be used because it doesn't have access to requester information, because if doesn't get Event object. And get_or_create_from_event could do populating of this info, but we are using get_or_create instead in places where processing new events. And those two functions do almost the same, but still they where created for something so I would let them be where they are now .

You can add these valid arguments to create and get_or_create, just like what you did for requester, for example:

    def get_or_create(cls, session, message_id, search_key, event_type,
                      released=True, manual=False, dry_run=False,
                      requester=None, requested_rebuilds=None,
                      requester_metadata=None):
    ...
    def get_or_create_from_event(cls, session, event, released=True):
        requester = getattr(event, "requester", None)
        requested_rebuilds = getattr(event, "container_images", None)
        if requested_rebuilds and isinstance(requested_rebuilds, list):
            requested_rebuilds = " ".join(requested_rebuilds)
        requester_metadata = getattr(event, "requester_metadata_json", None)

        return cls.get_or_create(
            session, event.msg_id,
            event.search_key, event.__class__,
            released=released, manual=event.manual,
            dry_run=event.dry_run,
            requester=requester,
            requested_rebuilds=requested_rebuilds,
            requester_metadata=requester_metadata)

which is easier to maintain.

1 new commit added

  • Implement populating of requester info for async.
4 years ago

get_or_create can't be used because it doesn't have access to requester information, because if doesn't get Event object. And get_or_create_from_event could do populating of this info, but we are using get_or_create instead in places where processing new events. And those two functions do almost the same, but still they where created for something so I would let them be where they are now .

You can add these valid arguments to create and get_or_create, just like what you did for requester, for example:
def get_or_create(cls, session, message_id, search_key, event_type,
released=True, manual=False, dry_run=False,
requester=None, requested_rebuilds=None,
requester_metadata=None):
...
def get_or_create_from_event(cls, session, event, released=True):
requester = getattr(event, "requester", None)
requested_rebuilds = getattr(event, "container_images", None)
if requested_rebuilds and isinstance(requested_rebuilds, list):
requested_rebuilds = " ".join(requested_rebuilds)
requester_metadata = getattr(event, "requester_metadata_json", None)

    return cls.get_or_create(
        session, event.msg_id,
        event.search_key, event.__class__,
        released=released, manual=event.manual,
        dry_run=event.dry_run,
        requester=requester,
        requested_rebuilds=requested_rebuilds,
        requester_metadata=requester_metadata)

which is easier to maintain.

I did a little improvement to my previous code, so now there is a function in Event class to populate requester fields it they are not filled yet. I think it could be better to have less optional parameters in get_or_create function to have it simpler to use and understand.

I did a little improvement to my previous code, so now there is a function in Event class to populate requester fields it they are not filled yet. I think it could be better to have less optional parameters in get_or_create function to have it simpler to use and understand.

With updating the Event's create functions, there will be no necessary to check event type and then fill the fields, and I think it makes sense to have these arguments in Event's create functions since they are columns of Event. What do you think?

I did a little improvement to my previous code, so now there is a function in Event class to populate requester fields it they are not filled yet. I think it could be better to have less optional parameters in get_or_create function to have it simpler to use and understand.

With updating the Event's create functions, there will be no necessary to check event type and then fill the fields, and I think it makes sense to have these arguments in Event's create functions since they are columns of Event. What do you think?

Yes i think you are right. I will change the code.

@qwan But I got it only now. If I will change get_or_create and get_or_create_from_event methods as you said and delete type checking. Still I will have to write this piece of code:
...
requester = getattr(event, "requester", None)
requested_rebuilds = getattr(event, "container_images", None)
if requested_rebuilds and isinstance(requested_rebuilds, list):
requested_rebuilds = " ".join(requested_rebuilds)
requester_metadata = getattr(event, "requester_metadata_json", None)
get_or_create(...)
...
everywhere where get_or_create method is used. Because it requires strings arguments for metadata and rebuilds. And I can't use get_or_create_from_event because their usage is a little bit different(e.g. we can't say to get_or_create_from_event method to make dry_run=False, as we do with get_or_create, same for released and manual fields).

So I am not sure what solution is better for this.

everywhere where get_or_create method is used. Because it requires strings arguments for metadata and rebuilds. And I can't use get_or_create_from_event because their usage is a little bit different(e.g. we can't say to get_or_create_from_event method to make dry_run=False, as we do with get_or_create, same for released and manual fields).

I believe we can change them (get_or_create) to get_or_create_from_event (at least for the ones in this PR), and event does have event.dry_run, event.manual, and you can pass released to get_or_create_from_event as well.

everywhere where get_or_create method is used. Because it requires strings arguments for metadata and rebuilds. And I can't use get_or_create_from_event because their usage is a little bit different(e.g. we can't say to get_or_create_from_event method to make dry_run=False, as we do with get_or_create, same for released and manual fields).

I believe we can change them (get_or_create) to get_or_create_from_event (at least for the ones in this PR), and event does have event.dry_run, event.manual, and you can pass released to get_or_create_from_event as well.

ok, then I will replace usages of get_or_create to get_or_create_from_event used in PR.

1 new commit added

  • Rewrite some functions for better maintenance
4 years ago

rebased onto fbfb82044c695d382b0569debc5d631642a9f15a

4 years ago

events always have manual and dry_run (check BaseEvent), so you don't need this.

requester_metadata has already been converted to string in views (_create_rebuild_event_from_request).

requester_metadata has already been converted to string in views (_create_rebuild_event_from_request).

I was wrong, ignore this please

The "container_images" can be "[]" (which is not None) in request, so this can break. We can just use

if requested_rebuilds_list and isinstance(requested_rebuilds_list, list):
    requested_rebuilds = " ".join(requested_rebuilds_list)
else:
    requested_rebuilds = None

The "container_images" can be "[]" (which is not None) in request, so this can break. We can just use if requested_rebuilds_list and ....

Yes but if it's empty list, it must be converted to string too, so thats why I have there 'is not None'

The "container_images" can be "[]" (which is not None) in request, so this can break. We can just use if requested_rebuilds_list and ....

Yes but if it's empty list, it must be converted to string too, so thats why I have there 'is not None'

ah, I see, you have set it to None by default, before the checking.

rebased onto aa45b2257a98fbbee7033b5385352a66b0f43ca7

4 years ago

Looks good to me too :+1:

rebased onto f07b8b4

4 years ago

Pull-Request has been merged by apaplaus

4 years ago