From 842754c2d348c1b3a4d15b58fa1a2cd6b5d03b83 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 07 2017 16:41:40 +0000 Subject: [PATCH 1/5] datetime compatibility option for callbacks --- diff --git a/koji/plugin.py b/koji/plugin.py index 0f874ef..6191f2e 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -25,6 +25,7 @@ import logging import sys import traceback import six +from koji.util import encode_datetime_recurse # the available callback hooks and a list # of functions to be called for each event @@ -165,6 +166,13 @@ def ignore_error(f): setattr(f, 'failure_is_an_option', True) return f + +def convert_datetime(f): + """Indicate that callback needs to receive datetime objects as strings""" + setattr(f, 'convert_datetime', True) + return f + + def register_callback(cbtype, func): if not cbtype in callbacks: raise koji.PluginError('"%s" is not a valid callback type' % cbtype) @@ -172,12 +180,15 @@ def register_callback(cbtype, func): raise koji.PluginError('%s is not callable' % getattr(func, '__name__', 'function')) callbacks[cbtype].append(func) + def run_callbacks(cbtype, *args, **kws): if not cbtype in callbacks: raise koji.PluginError('"%s" is not a valid callback type' % cbtype) + cache = {} for func in callbacks[cbtype]: + cb_args, cb_kwargs = _fix_cb_args(func, args, kws, cache) try: - func(cbtype, *args, **kws) + func(cbtype, *cb_args, **cb_kwargs) except: msg = 'Error running %s callback from %s' % (cbtype, func.__module__) if getattr(func, 'failure_is_an_option', False): @@ -185,3 +196,10 @@ def run_callbacks(cbtype, *args, **kws): else: tb = ''.join(traceback.format_exception(*sys.exc_info())) raise koji.CallbackError('%s:\n%s' % (msg, tb)) + + +def _fix_cb_args(func, args, kwargs, cache): + if getattr(func, 'convert_datetime', False): + args = encode_datetime_recurse(args) + kwargs = encode_datetime_recurse(kwargs) + return args, kwargs diff --git a/koji/util.py b/koji/util.py index 2cde6c6..7d80504 100644 --- a/koji/util.py +++ b/koji/util.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import calendar +import datetime from fnmatch import fnmatch import koji import logging @@ -151,6 +152,51 @@ def dslice_ex(dict, keys, strict=True): del ret[key] return ret + +class DataWalker(object): + + def __init__(self, data, callback, kwargs=None): + self.data = data + self.callback = callback + if kwargs is None: + kwargs = {} + self.kwargs = kwargs + + def walk(self): + return self._walk(self.data) + + def _walk(self, value): + # first let callback filter the value + value = self.callback(value, **self.kwargs) + # then recurse if needed + if isinstance(value, tuple): + return tuple([self._walk(x) for x in value]) + elif isinstance(value, list): + return list([self._walk(x) for x in value]) + elif isinstance(value, dict): + ret = {} + for k in value: + k = self._walk(k) + v = self._walk(value[k]) + ret[k] = v + return ret + else: + return value + + +def encode_datetime(value): + """Convert datetime objects to strings""" + if isinstance(value, datetime.datetime): + return value.isoformat(' ') + else: + return value + + +def encode_datetime_recurse(value): + walker = DataWalker(value, encode_datetime) + return walker.walk() + + def call_with_argcheck(func, args, kwargs=None): """Call function, raising ParameterError if args do not match""" if kwargs is None: From 1df193ea2b65ca1115e97046cb24b75f320c7ae7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 07 2017 16:41:40 +0000 Subject: [PATCH 2/5] use datetime compatibility for messagebug and protonmsg plugins --- diff --git a/plugins/hub/messagebus.py b/plugins/hub/messagebus.py index 10c8b00..b09cccb 100644 --- a/plugins/hub/messagebus.py +++ b/plugins/hub/messagebus.py @@ -5,7 +5,7 @@ # Mike Bonnet from koji import PluginError -from koji.plugin import callbacks, callback, ignore_error +from koji.plugin import callbacks, callback, ignore_error, convert_datetime import ConfigParser import logging import qpid.messaging @@ -203,6 +203,7 @@ def get_message_headers(msgtype, *args, **kws): @callback(*[c for c in callbacks.keys() if c.startswith('post') and c != 'postCommit']) @ignore_error +@convert_datetime def send_message(cbtype, *args, **kws): global config sender = get_sender() diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index cf3a82c..f91f667 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -6,7 +6,7 @@ # Mike Bonnet import koji -from koji.plugin import callback, ignore_error +from koji.plugin import callback, ignore_error, convert_datetime from koji.context import context import ConfigParser import logging @@ -138,6 +138,7 @@ def queue_msg(address, props, data): body = json.dumps(data) msgs.append((address, props, body)) +@convert_datetime @callback('postPackageListChange') def prep_package_list_change(cbtype, *args, **kws): address = 'package.' + kws['action'] @@ -147,6 +148,7 @@ def prep_package_list_change(cbtype, *args, **kws): 'action': kws['action']} queue_msg(address, props, kws) +@convert_datetime @callback('postTaskStateChange') def prep_task_state_change(cbtype, *args, **kws): if kws['attribute'] != 'state': @@ -161,6 +163,7 @@ def prep_task_state_change(cbtype, *args, **kws): 'new': kws['new']} queue_msg(address, props, kws) +@convert_datetime @callback('postBuildStateChange') def prep_build_state_change(cbtype, *args, **kws): if kws['attribute'] != 'state': @@ -179,6 +182,7 @@ def prep_build_state_change(cbtype, *args, **kws): 'new': new} queue_msg(address, props, kws) +@convert_datetime @callback('postImport') def prep_import(cbtype, *args, **kws): address = 'import.' + kws['type'] @@ -189,6 +193,7 @@ def prep_import(cbtype, *args, **kws): 'release': kws['build']['release']} queue_msg(address, props, kws) +@convert_datetime @callback('postRPMSign') def prep_rpm_sign(cbtype, *args, **kws): address = 'sign.rpm' @@ -212,14 +217,17 @@ def _prep_tag_msg(address, cbtype, kws): 'user': kws['user']['name']} queue_msg(address, props, kws) +@convert_datetime @callback('postTag') def prep_tag(cbtype, *args, **kws): _prep_tag_msg('build.tag', cbtype, kws) +@convert_datetime @callback('postUntag') def prep_untag(cbtype, *args, **kws): _prep_tag_msg('build.untag', cbtype, kws) +@convert_datetime @callback('postRepoInit') def prep_repo_init(cbtype, *args, **kws): address = 'repo.init' @@ -228,6 +236,7 @@ def prep_repo_init(cbtype, *args, **kws): 'repo_id': kws['repo_id']} queue_msg(address, props, kws) +@convert_datetime @callback('postRepoDone') def prep_repo_done(cbtype, *args, **kws): address = 'repo.done' @@ -238,6 +247,7 @@ def prep_repo_done(cbtype, *args, **kws): queue_msg(address, props, kws) @ignore_error +@convert_datetime @callback('postCommit') def send_queued_msgs(cbtype, *args, **kws): msgs = getattr(context, 'protonmsg_msgs', None) From ab928e6196678069fe601cae47cb726ae9225e36 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 07 2017 16:41:40 +0000 Subject: [PATCH 3/5] add unit test for convert_datetime decorator --- diff --git a/tests/test_lib/test_plugin.py b/tests/test_lib/test_plugin.py index 02376af..ea24be3 100644 --- a/tests/test_lib/test_plugin.py +++ b/tests/test_lib/test_plugin.py @@ -1,8 +1,10 @@ import copy +import datetime import mock import unittest import koji +import koji.util import koji.plugin @@ -98,6 +100,10 @@ class TestCallbacks(unittest.TestCase): self.callbacks.append([cbtype, args, kwargs]) raise TestError + @koji.plugin.convert_datetime + def datetime_callback(self, cbtype, *args, **kwargs): + self.callbacks.append([cbtype, args, kwargs]) + def test_simple_callback(self): args = ('hello',) kwargs = {'world': 1} @@ -127,6 +133,19 @@ class TestCallbacks(unittest.TestCase): getLogger.assert_called_once() getLogger.return_value.warn.assert_called_once() + def test_datetime_callback(self): + dt1 = datetime.datetime.now() + dt2 = datetime.datetime(2001,1,1) + args = (dt1,"2",["three"], {4: dt2},) + kwargs = {'foo': [dt1, dt2]} + cbtype = 'preTag' + koji.plugin.register_callback(cbtype, self.datetime_callback) + koji.plugin.run_callbacks(cbtype, *args, **kwargs) + args2 = koji.util.encode_datetime_recurse(args) + kwargs2 = koji.util.encode_datetime_recurse(kwargs) + self.assertEqual(len(self.callbacks), 1) + self.assertEqual(self.callbacks[0], [cbtype, args2, kwargs2]) + def test_bad_callback(self): args = ('hello',) kwargs = {'world': 1} From fe7c0b4052eee8027a51832d0bda47b5def1602d Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 07 2017 16:41:40 +0000 Subject: [PATCH 4/5] unit test for encode_datetime --- diff --git a/tests/test_lib/test_encode_datetime.py b/tests/test_lib/test_encode_datetime.py new file mode 100644 index 0000000..7617c26 --- /dev/null +++ b/tests/test_lib/test_encode_datetime.py @@ -0,0 +1,37 @@ +import datetime +import unittest + +import koji.util + + +class testEncodeDatetime(unittest.TestCase): + + DATES = [ + [datetime.datetime(2001, 2, 3, 9, 45, 32), + '2001-02-03 09:45:32'], + [datetime.datetime(1970, 1, 1, 0, 0), + '1970-01-01 00:00:00'], + [datetime.datetime(2017, 8, 3, 10, 19, 39, 474556), + '2017-08-03 10:19:39.474556'], + ] + + def test_simple_dates(self): + for dt, dstr in self.DATES: + chk1 = koji.util.encode_datetime(dt) + chk2 = koji.util.encode_datetime_recurse(dt) + self.assertEqual(chk1, dstr) + self.assertEqual(chk2, dstr) + + + def test_embedded_dates(self): + dt1, ds1 = self.DATES[0] + dt2, ds2 = self.DATES[1] + dt3, ds3 = self.DATES[2] + data1 = [1, "2", [3, dt1], {"4": dt2}, [[[[{"five": dt3}]]]]] + fix_1 = [1, "2", [3, ds1], {"4": ds2}, [[[[{"five": ds3}]]]]] + data2 = {1: dt1, "2": [dt2, dt1], "three": {"3": {3: dt3}}} + fix_2 = {1: ds1, "2": [ds2, ds1], "three": {"3": {3: ds3}}} + chk1 = koji.util.encode_datetime_recurse(data1) + chk2 = koji.util.encode_datetime_recurse(data2) + self.assertEqual(chk1, fix_1) + self.assertEqual(chk2, fix_2) From 0c8d4c8f07096f5714d2407cc3f4e36a3be1509c Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 07 2017 16:41:40 +0000 Subject: [PATCH 5/5] implement caching in _fix_cb_args --- diff --git a/koji/plugin.py b/koji/plugin.py index 6191f2e..396b811 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -200,6 +200,16 @@ def run_callbacks(cbtype, *args, **kws): def _fix_cb_args(func, args, kwargs, cache): if getattr(func, 'convert_datetime', False): - args = encode_datetime_recurse(args) - kwargs = encode_datetime_recurse(kwargs) + if id(args) in cache: + args = cache[id(args)] + else: + val = encode_datetime_recurse(args) + cache[id(args)] = val + args = val + if id(kwargs) in cache: + kwargs = cache[id(kwargs)] + else: + val = encode_datetime_recurse(kwargs) + cache[id(kwargs)] = val + kwargs = val return args, kwargs diff --git a/tests/test_lib/test_plugin.py b/tests/test_lib/test_plugin.py index ea24be3..f064a5d 100644 --- a/tests/test_lib/test_plugin.py +++ b/tests/test_lib/test_plugin.py @@ -104,6 +104,10 @@ class TestCallbacks(unittest.TestCase): def datetime_callback(self, cbtype, *args, **kwargs): self.callbacks.append([cbtype, args, kwargs]) + @koji.plugin.convert_datetime + def datetime_callback2(self, cbtype, *args, **kwargs): + self.callbacks.append([cbtype, args, kwargs]) + def test_simple_callback(self): args = ('hello',) kwargs = {'world': 1} @@ -146,6 +150,34 @@ class TestCallbacks(unittest.TestCase): self.assertEqual(len(self.callbacks), 1) self.assertEqual(self.callbacks[0], [cbtype, args2, kwargs2]) + def test_multiple_datetime_callback(self): + dt1 = datetime.datetime.now() + dt2 = datetime.datetime(2001,1,1) + args = (dt1,"2",["three"], {4: dt2},) + kwargs = {'foo': [dt1, dt2]} + cbtype = 'preTag' + koji.plugin.register_callback(cbtype, self.datetime_callback) + koji.plugin.register_callback(cbtype, self.datetime_callback2) + koji.plugin.run_callbacks(cbtype, *args, **kwargs) + args2 = koji.util.encode_datetime_recurse(args) + kwargs2 = koji.util.encode_datetime_recurse(kwargs) + self.assertEqual(len(self.callbacks), 2) + self.assertEqual(self.callbacks[0], [cbtype, args2, kwargs2]) + self.assertEqual(self.callbacks[1], [cbtype, args2, kwargs2]) + # verify that caching worked + # unfortunately, args and kwargs get unpacked and repacked, so we have + # to dig down + cb_args1 = self.callbacks[0][1] + cb_args2 = self.callbacks[1][1] + for i in range(len(cb_args1)): + if cb_args1[i] is not cb_args2[i]: + raise Exception("converted args not cached") + cb_kwargs1 = self.callbacks[0][2] + cb_kwargs2 = self.callbacks[1][2] + for k in cb_kwargs1: + if cb_kwargs1[k] is not cb_kwargs2[k]: + raise Exception("converted kwargs not cached") + def test_bad_callback(self): args = ('hello',) kwargs = {'world': 1}