koji

Clone

#535 datetime compatibility for plugins
Merged 6 years ago by mikem. Opened 6 years ago by mikem.
mikem/koji datetime-compat-plugins  into  master

file modified
+29 -1 
@@ -25,6 +25,7 @@ 
 

  import sys
 

  import traceback
 

  import six
 

+ from koji.util import encode_datetime_recurse
 

  

  # the available callback hooks and a list
 

  # of functions to be called for each event
 
@@ -165,6 +166,13 @@ 
 

      setattr(f, 'failure_is_an_option', True)
 

      return f
 

  

+ 

+ def convert_datetime(f):
 

+     """Indicate that callback needs to receive datetime objects as strings"""
 

+     setattr(f, 'convert_datetime', True)
 

+     return f
 

+ 

+ 

  def register_callback(cbtype, func):
 

      if not cbtype in callbacks:
 

          raise koji.PluginError('"%s" is not a valid callback type' % cbtype)
 
@@ -172,12 +180,15 @@ 
 

          raise koji.PluginError('%s is not callable' % getattr(func, '__name__', 'function'))
 

      callbacks[cbtype].append(func)
 

  

+ 

  def run_callbacks(cbtype, *args, **kws):
 

      if not cbtype in callbacks:
 

          raise koji.PluginError('"%s" is not a valid callback type' % cbtype)
 

+     cache = {}
 

      for func in callbacks[cbtype]:
 

+         cb_args, cb_kwargs = _fix_cb_args(func, args, kws, cache)
 

          try:
 

-             func(cbtype, *args, **kws)
 

+             func(cbtype, *cb_args, **cb_kwargs)
 

          except:
 

              msg = 'Error running %s callback from %s' % (cbtype, func.__module__)
 

              if getattr(func, 'failure_is_an_option', False):
 
@@ -185,3 +196,20 @@ 
 

              else:
 

                  tb = ''.join(traceback.format_exception(*sys.exc_info()))
 

                  raise koji.CallbackError('%s:\n%s' % (msg, tb))
 

+ 

+ 

+ def _fix_cb_args(func, args, kwargs, cache):
 

+     if getattr(func, 'convert_datetime', False):
 

+         if id(args) in cache:
 

+             args = cache[id(args)]
 

+         else:
 

+             val = encode_datetime_recurse(args)
 

+             cache[id(args)] = val
 

+             args = val
 

+         if id(kwargs) in cache:
 

+             kwargs = cache[id(kwargs)]
 

+         else:
 

+             val = encode_datetime_recurse(kwargs)
 

+             cache[id(kwargs)] = val
 

+             kwargs = val
 

+     return args, kwargs

file modified
+46 
@@ -20,6 +20,7 @@ 
 

  

  from __future__ import absolute_import
 

  import calendar
 

+ import datetime
 

  from fnmatch import fnmatch
 

  import koji
 

  import logging
 
@@ -151,6 +152,51 @@ 
 

              del ret[key]
 

      return ret
 

  

+ 

+ class DataWalker(object):
 

+ 

+     def __init__(self, data, callback, kwargs=None):
 

+         self.data = data
 

+         self.callback = callback
 

+         if kwargs is None:
 

+             kwargs = {}
 

+         self.kwargs = kwargs
 

+ 

+     def walk(self):
 

+         return self._walk(self.data)
 

+ 

+     def _walk(self, value):
 

+         # first let callback filter the value
 

+         value = self.callback(value, **self.kwargs)
 

+         # then recurse if needed
 

+         if isinstance(value, tuple):
 

+             return tuple([self._walk(x) for x in value])
 

+         elif isinstance(value, list):
 

+             return list([self._walk(x) for x in value])
 

+         elif isinstance(value, dict):
 

+             ret = {}
 

+             for k in value:
 

+                 k = self._walk(k)
 

+                 v = self._walk(value[k])
 

+                 ret[k] = v
 

+             return ret
 

+         else:
 

+             return value
 

+ 

+ 

+ def encode_datetime(value):
 

+     """Convert datetime objects to strings"""
 

+     if isinstance(value, datetime.datetime):
 

+         return value.isoformat(' ')
 

+     else:
 

+         return value
 

+ 

+ 

+ def encode_datetime_recurse(value):
 

+     walker = DataWalker(value, encode_datetime)
 

+     return walker.walk()
 

+ 

+ 

  def call_with_argcheck(func, args, kwargs=None):
 

      """Call function, raising ParameterError if args do not match"""
 

      if kwargs is None:

file modified
+2 -1 
@@ -5,7 +5,7 @@ 
 

  #     Mike Bonnet <mikeb@redhat.com>
 

  

  from koji import PluginError
 

- from koji.plugin import callbacks, callback, ignore_error
 

+ from koji.plugin import callbacks, callback, ignore_error, convert_datetime
 

  import ConfigParser
 

  import logging
 

  import qpid.messaging
 
@@ -203,6 +203,7 @@ 
 

  @callback(*[c for c in callbacks.keys() if c.startswith('post')
 

              and c != 'postCommit'])
 

  @ignore_error
 

+ @convert_datetime
 

  def send_message(cbtype, *args, **kws):
 

      global config
 

      sender = get_sender()

file modified
+11 -1 
@@ -6,7 +6,7 @@ 
 

  #     Mike Bonnet <mikeb@redhat.com>
 

  

  import koji
 

- from koji.plugin import callback, ignore_error
 

+ from koji.plugin import callback, ignore_error, convert_datetime
 

  from koji.context import context
 

  import ConfigParser
 

  import logging
 
@@ -138,6 +138,7 @@ 
 

      body = json.dumps(data)
 

      msgs.append((address, props, body))
 

  

+ @convert_datetime
 

  @callback('postPackageListChange')
 

  def prep_package_list_change(cbtype, *args, **kws):
 

      address = 'package.' + kws['action']
 
@@ -147,6 +148,7 @@ 
 

               'action': kws['action']}
 

      queue_msg(address, props, kws)
 

  

+ @convert_datetime
 

  @callback('postTaskStateChange')
 

  def prep_task_state_change(cbtype, *args, **kws):
 

      if kws['attribute'] != 'state':
 
@@ -161,6 +163,7 @@ 
 

               'new': kws['new']}
 

      queue_msg(address, props, kws)
 

  

+ @convert_datetime
 

  @callback('postBuildStateChange')
 

  def prep_build_state_change(cbtype, *args, **kws):
 

      if kws['attribute'] != 'state':
 
@@ -179,6 +182,7 @@ 
 

               'new': new}
 

      queue_msg(address, props, kws)
 

  

+ @convert_datetime
 

  @callback('postImport')
 

  def prep_import(cbtype, *args, **kws):
 

      address = 'import.' + kws['type']
 
@@ -189,6 +193,7 @@ 
 

               'release': kws['build']['release']}
 

      queue_msg(address, props, kws)
 

  

+ @convert_datetime
 

  @callback('postRPMSign')
 

  def prep_rpm_sign(cbtype, *args, **kws):
 

      address = 'sign.rpm'
 
@@ -212,14 +217,17 @@ 
 

               'user': kws['user']['name']}
 

      queue_msg(address, props, kws)
 

  

+ @convert_datetime
 

  @callback('postTag')
 

  def prep_tag(cbtype, *args, **kws):
 

      _prep_tag_msg('build.tag', cbtype, kws)
 

  

+ @convert_datetime
 

  @callback('postUntag')
 

  def prep_untag(cbtype, *args, **kws):
 

      _prep_tag_msg('build.untag', cbtype, kws)
 

  

+ @convert_datetime
 

  @callback('postRepoInit')
 

  def prep_repo_init(cbtype, *args, **kws):
 

      address = 'repo.init'
 
@@ -228,6 +236,7 @@ 
 

               'repo_id': kws['repo_id']}
 

      queue_msg(address, props, kws)
 

  

+ @convert_datetime
 

  @callback('postRepoDone')
 

  def prep_repo_done(cbtype, *args, **kws):
 

      address = 'repo.done'
 
@@ -238,6 +247,7 @@ 
 

      queue_msg(address, props, kws)
 

  

  @ignore_error
 

+ @convert_datetime
 

  @callback('postCommit')
 

  def send_queued_msgs(cbtype, *args, **kws):
 

      msgs = getattr(context, 'protonmsg_msgs', None)

file added
+37 
@@ -0,0 +1,37 @@ 
 

+ import datetime
 

+ import unittest
 

+ 

+ import koji.util
 

+ 

+ 

+ class testEncodeDatetime(unittest.TestCase):
 

+ 

+     DATES = [
 

+             [datetime.datetime(2001, 2, 3, 9, 45, 32),
 

+                 '2001-02-03 09:45:32'],
 

+             [datetime.datetime(1970, 1, 1, 0, 0),
 

+                 '1970-01-01 00:00:00'],
 

+             [datetime.datetime(2017, 8, 3, 10, 19, 39, 474556),
 

+                 '2017-08-03 10:19:39.474556'],
 

+           ]
 

+ 

+     def test_simple_dates(self):
 

+         for dt, dstr in self.DATES:
 

+             chk1 = koji.util.encode_datetime(dt)
 

+             chk2 = koji.util.encode_datetime_recurse(dt)
 

+             self.assertEqual(chk1, dstr)
 

+             self.assertEqual(chk2, dstr)
 

+ 

+ 

+     def test_embedded_dates(self):
 

+         dt1, ds1 = self.DATES[0]
 

+         dt2, ds2 = self.DATES[1]
 

+         dt3, ds3 = self.DATES[2]
 

+         data1 = [1, "2", [3, dt1], {"4": dt2}, [[[[{"five": dt3}]]]]]
 

+         fix_1 = [1, "2", [3, ds1], {"4": ds2}, [[[[{"five": ds3}]]]]]
 

+         data2 = {1: dt1, "2": [dt2, dt1], "three": {"3": {3: dt3}}}
 

+         fix_2 = {1: ds1, "2": [ds2, ds1], "three": {"3": {3: ds3}}}
 

+         chk1 = koji.util.encode_datetime_recurse(data1)
 

+         chk2 = koji.util.encode_datetime_recurse(data2)
 

+         self.assertEqual(chk1, fix_1)
 

+         self.assertEqual(chk2, fix_2)

file modified
+51 
@@ -1,8 +1,10 @@ 
 

  import copy
 

+ import datetime
 

  import mock
 

  import unittest
 

  

  import koji
 

+ import koji.util
 

  import koji.plugin
 

  

  
@@ -98,6 +100,14 @@ 
 

          self.callbacks.append([cbtype, args, kwargs])
 

          raise TestError
 

  

+     @koji.plugin.convert_datetime
 

+     def datetime_callback(self, cbtype, *args, **kwargs):
 

+         self.callbacks.append([cbtype, args, kwargs])
 

+ 

+     @koji.plugin.convert_datetime
 

+     def datetime_callback2(self, cbtype, *args, **kwargs):
 

+         self.callbacks.append([cbtype, args, kwargs])
 

+ 

      def test_simple_callback(self):
 

          args = ('hello',)
 

          kwargs = {'world': 1}
 
@@ -127,6 +137,47 @@ 
 

          getLogger.assert_called_once()
 

          getLogger.return_value.warn.assert_called_once()
 

  

+     def test_datetime_callback(self):
 

+         dt1 = datetime.datetime.now()
 

+         dt2 = datetime.datetime(2001,1,1)
 

+         args = (dt1,"2",["three"], {4: dt2},)
 

+         kwargs = {'foo': [dt1, dt2]}
 

+         cbtype = 'preTag'
 

+         koji.plugin.register_callback(cbtype, self.datetime_callback)
 

+         koji.plugin.run_callbacks(cbtype, *args, **kwargs)
 

+         args2 = koji.util.encode_datetime_recurse(args)
 

+         kwargs2 = koji.util.encode_datetime_recurse(kwargs)
 

+         self.assertEqual(len(self.callbacks), 1)
 

+         self.assertEqual(self.callbacks[0], [cbtype, args2, kwargs2])
 

+ 

+     def test_multiple_datetime_callback(self):
 

+         dt1 = datetime.datetime.now()
 

+         dt2 = datetime.datetime(2001,1,1)
 

+         args = (dt1,"2",["three"], {4: dt2},)
 

+         kwargs = {'foo': [dt1, dt2]}
 

+         cbtype = 'preTag'
 

+         koji.plugin.register_callback(cbtype, self.datetime_callback)
 

+         koji.plugin.register_callback(cbtype, self.datetime_callback2)
 

+         koji.plugin.run_callbacks(cbtype, *args, **kwargs)
 

+         args2 = koji.util.encode_datetime_recurse(args)
 

+         kwargs2 = koji.util.encode_datetime_recurse(kwargs)
 

+         self.assertEqual(len(self.callbacks), 2)
 

+         self.assertEqual(self.callbacks[0], [cbtype, args2, kwargs2])
 

+         self.assertEqual(self.callbacks[1], [cbtype, args2, kwargs2])
 

+         # verify that caching worked
 

+         # unfortunately, args and kwargs get unpacked and repacked, so we have
 

+         # to dig down
 

+         cb_args1 = self.callbacks[0][1]
 

+         cb_args2 = self.callbacks[1][1]
 

+         for i in range(len(cb_args1)):
 

+             if cb_args1[i] is not cb_args2[i]:
 

+                 raise Exception("converted args not cached")
 

+         cb_kwargs1 = self.callbacks[0][2]
 

+         cb_kwargs2 = self.callbacks[1][2]
 

+         for k in cb_kwargs1:
 

+             if cb_kwargs1[k] is not cb_kwargs2[k]:
 

+                 raise Exception("converted kwargs not cached")
 

+ 

      def test_bad_callback(self):
 

          args = ('hello',)
 

          kwargs = {'world': 1}

Some plugins might not like the datetime objects that we now have internally on the hub (thanks to psycopg2). This provides the option to automatically convert them to strings (as they were before 1.12).

We use this option for the included protonmsg and messagebus plugins.

2 new commits added

  • unit test for encode_datetime
  • add unit test for convert_datetime decorator
6 years ago

LGTM

1 new commit added

  • implement caching in _fix_cb_args
6 years ago

rebased
6 years ago

Commit 4a05467 fixes this pull-request

Pull-Request has been merged by mikem@redhat.com
6 years ago
Changes Summary 6
+29 -1
file changed
koji/plugin.py
+46 -0
file changed
koji/util.py
+2 -1
file changed
plugins/hub/messagebus.py
+11 -1
file changed
plugins/hub/protonmsg.py
+37
file added
tests/test_lib/test_encode_datetime.py
+51 -0
file changed
tests/test_lib/test_plugin.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.