#2047 limit size of extra field in proton msgs
Merged 2 years ago by tkopecek. Opened 2 years ago by tkopecek.
tkopecek/koji issue2017  into  master

@@ -5,3 +5,9 @@ 

  topic_prefix = koji

  connect_timeout = 10

  send_timeout = 60

+ 

+ [message]

+ # limit build.extra size which is sent to bus

+ # if field is longer (json.dumps), ignore it

+ # default value is 0 - unlimited size

+ extra_limit = 0

file modified
+22 -4
@@ -135,6 +135,21 @@ 

              self.timeout_task = None

  

  

+ def _strip_extra(buildinfo):

+     """If extra_limit is configured, compare extra's size and drop it,

+     if it is over"""

+     global CONFIG

+     if not CONFIG:

+         CONFIG = koji.read_config_files([(CONFIG_FILE, True)])

+     if CONFIG.has_option('message', 'extra_limit'):

+         extra_limit = CONFIG.getint('message', 'extra_limit')

+         extra_size = len(json_serialize(buildinfo.get('extra', {})))

+         if extra_limit and extra_size > extra_limit:

+             buildinfo = buildinfo.copy()

+             del buildinfo['extra']

+     return buildinfo

+ 

+ 

  def json_serialize(o):

      """JSON helper to encode otherwise unserializable data types"""

      if isinstance(o, set):
@@ -191,6 +206,7 @@ 

          old = koji.BUILD_STATES[old]

      new = koji.BUILD_STATES[kws['new']]

      address = 'build.' + new.lower()

+     kws['info'] = _strip_extra(kws['info'])

      props = {'type': cbtype[4:],

               'name': kws['info']['name'],

               'version': kws['info']['version'],
@@ -204,6 +220,7 @@ 

  @convert_datetime

  @callback('postImport')

  def prep_import(cbtype, *args, **kws):

+     kws['build'] = _strip_extra(kws['build'])

      address = 'import.' + kws['type']

      props = {'type': cbtype[4:],

               'importType': kws['type'],
@@ -218,6 +235,7 @@ 

  def prep_rpm_sign(cbtype, *args, **kws):

      if not kws['sigkey']:

          return

+     kws['build'] = _strip_extra(kws['build'])

      address = 'sign.rpm'

      props = {'type': cbtype[4:],

               'sigkey': kws['sigkey'],
@@ -232,12 +250,12 @@ 

  

  

  def _prep_tag_msg(address, cbtype, kws):

-     build = kws['build']

+     kws['build'] = _strip_extra(kws['build'])

      props = {'type': cbtype[4:],

               'tag': kws['tag']['name'],

-              'name': build['name'],

-              'version': build['version'],

-              'release': build['release'],

+              'name': kws['build']['name'],

+              'version': kws['build']['version'],

+              'release': kws['build']['release'],

               'user': kws['user']['name']}

      queue_msg(address, props, kws)

  

@@ -12,9 +12,27 @@ 

  from six.moves.configparser import ConfigParser, SafeConfigParser

  

  class TestProtonMsg(unittest.TestCase):

+     def setUp(self):

+         self.conf = tempfile.NamedTemporaryFile()

+         self.conf.write(six.b("""[broker]

+ urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671

+ cert = /etc/koji-hub/plugins/client.pem

+ cacert = /etc/koji-hub/plugins/ca.pem

+ topic_prefix = koji

+ connect_timeout = 10

+ send_timeout = 60

+ 

+ [message]

+ extra_limit = 2048

+ """))

+         self.conf.flush()

+         protonmsg.CONFIG_FILE = self.conf.name

+         protonmsg.CONFIG = None

+ 

      def tearDown(self):

          if hasattr(context, 'protonmsg_msgs'):

              del context.protonmsg_msgs

+         del self.conf

  

      def assertMsg(self, topic, body=None, **kws):

          self.assertTrue(hasattr(context, 'protonmsg_msgs'))
@@ -191,18 +209,6 @@ 

      @patch('logging.getLogger')

      def test_send_queued_msgs_fail(self, getLogger, Container):

          context.protonmsg_msgs = [('test.topic', {'testheader': 1}, 'test body')]

-         conf = tempfile.NamedTemporaryFile()

-         conf.write(six.b("""[broker]

- urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671

- cert = /etc/koji-hub/plugins/client.pem

- cacert = /etc/koji-hub/plugins/ca.pem

- topic_prefix = koji

- connect_timeout = 10

- send_timeout = 60

- """))

-         conf.flush()

-         protonmsg.CONFIG_FILE = conf.name

-         protonmsg.CONFIG = None

          protonmsg.send_queued_msgs('postCommit')

          log = getLogger.return_value

          self.assertEqual(log.debug.call_count, 2)
@@ -215,18 +221,6 @@ 

      @patch('logging.getLogger')

      def test_send_queued_msgs_success(self, getLogger, Container):

          context.protonmsg_msgs = [('test.topic', {'testheader': 1}, 'test body')]

-         conf = tempfile.NamedTemporaryFile()

-         conf.write(six.b("""[broker]

- urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671

- cert = /etc/koji-hub/plugins/client.pem

- cacert = /etc/koji-hub/plugins/ca.pem

- topic_prefix = koji

- connect_timeout = 10

- send_timeout = 60

- """))

-         conf.flush()

-         protonmsg.CONFIG_FILE = conf.name

-         protonmsg.CONFIG = None

          def clear_msgs():

              del context.protonmsg_msgs[:]

          Container.return_value.run.side_effect = clear_msgs

rebased onto 6705922f04468935983a6ed84a66666360238c18

2 years ago

pretty please pagure-ci rebuild

2 years ago

rebased onto 464baf5

2 years ago

We should not modify the buildinfo values. They are passed in by reference in these callbacks.

I do agree with defaulting to no limit

1 new commit added

  • use buildinfo copy
2 years ago

Metadata Update from @tkopecek:
- Pull-request tagged with: testing-ready

2 years ago

Commit 6caa66f fixes this pull-request

Pull-Request has been merged by tkopecek

2 years ago

Metadata Update from @jcupova:
- Pull-request tagged with: testing-done

2 years ago