From 80ca9ff294487367ff1b2b7307bf0963edaa5cc1 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Oct 11 2017 20:36:40 +0000 Subject: [PATCH 1/2] handle callbacks depth-first in DataWalker --- diff --git a/koji/util.py b/koji/util.py index 7d80504..ba999a7 100644 --- a/koji/util.py +++ b/koji/util.py @@ -166,22 +166,20 @@ class DataWalker(object): return self._walk(self.data) def _walk(self, value): - # first let callback filter the value - value = self.callback(value, **self.kwargs) - # then recurse if needed + # recurse if needed if isinstance(value, tuple): - return tuple([self._walk(x) for x in value]) + value = tuple([self._walk(x) for x in value]) elif isinstance(value, list): - return list([self._walk(x) for x in value]) + value = list([self._walk(x) for x in value]) elif isinstance(value, dict): ret = {} for k in value: k = self._walk(k) v = self._walk(value[k]) ret[k] = v - return ret - else: - return value + value = ret + # finally, let callback filter the value + return self.callback(value, **self.kwargs) def encode_datetime(value): From 00e059c0c0f4a5dac535b2d53754ef0768c8c902 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Oct 11 2017 20:36:40 +0000 Subject: [PATCH 2/2] unit test for buildNotification task handler --- diff --git a/tests/test_builder/data/calls/build_notif_1/calls.json b/tests/test_builder/data/calls/build_notif_1/calls.json new file mode 100644 index 0000000..e71dd2e --- /dev/null +++ b/tests/test_builder/data/calls/build_notif_1/calls.json @@ -0,0 +1,362 @@ +[ + { + "method": "getTaskInfo", + "args": [ + 8982172 + ], + "kwargs": {}, + "result": { + "weight": 0.2, + "parent": null, + "completion_time": "2015-02-18 14:57:37.719876", + "start_time": "2015-02-18 14:45:50.535914", + "start_ts": 1424270750.53591, + "state": 2, + "awaited": null, + "label": null, + "priority": 20, + "channel_id": 1, + "waiting": false, + "create_time": "2015-02-18 14:45:43.283233", + "id": 8982172, + "create_ts": 1424270743.28323, + "owner": 2045, + "host_id": 84, + "completion_ts": 1424271457.71988, + "arch": "noarch", + "method": "build" + } + }, + { + "method": "getHost", + "args": [ + 84 + ], + "kwargs": {}, + "result": { + "comment": null, + "capacity": 4.0, + "name": "buildvm-26.build.example.com", + "enabled": true, + "arches": "i386 x86_64", + "task_load": 0.0, + "ready": true, + "user_id": 2179, + "id": 84, + "description": null + } + }, + { + "method": "getTaskResult", + "args": [ + 8982172 + ], + "kwargs": {}, + "result": null + }, + { + "method": "listTaskOutput", + "args": [ + 8982172 + ], + "kwargs": { + "all_volumes": true + }, + "result": {} + }, + { + "method": "getTaskRequest", + "args": [ + 8982172 + ], + "kwargs": {}, + "result": [ + "git://pkgs.example.com/sisu?#5428a6665ed5d4920bda5da6db6c9fce942070ac", + "rawhide", + { + "skip_tag": true + } + ] + }, + { + "method": "getTaskChildren", + "args": [ + 8982172 + ], + "kwargs": {}, + "result": [ + { + "weight": 1.0, + "parent": 8982172, + "completion_time": "2015-02-18 14:50:13.119183", + "start_time": "2015-02-18 14:45:59.156319", + "start_ts": 1424270759.15632, + "state": 2, + "awaited": false, + "label": "srpm", + "priority": 19, + "channel_id": 1, + "waiting": null, + "create_time": "2015-02-18 14:45:51.418503", + "id": 8982174, + "create_ts": 1424270751.4185, + "owner": 2045, + "host_id": 116, + "completion_ts": 1424271013.11918, + "arch": "noarch", + "method": "buildSRPMFromSCM" + }, + { + "weight": 1.62572677375, + "parent": 8982172, + "completion_time": "2015-02-18 14:57:33.063121", + "start_time": "2015-02-18 14:50:43.011525", + "start_ts": 1424271043.01152, + "state": 2, + "awaited": false, + "label": "noarch", + "priority": 19, + "channel_id": 1, + "waiting": null, + "create_time": "2015-02-18 14:50:37.546755", + "id": 8982231, + "create_ts": 1424271037.54676, + "owner": 2045, + "host_id": 81, + "completion_ts": 1424271453.06312, + "arch": "noarch", + "method": "buildArch" + } + ] + }, + { + "method": "getTaskInfo", + "args": [ + 8982174 + ], + "kwargs": {}, + "result": { + "weight": 1.0, + "parent": 8982172, + "completion_time": "2015-02-18 14:50:13.119183", + "start_time": "2015-02-18 14:45:59.156319", + "start_ts": 1424270759.15632, + "state": 2, + "awaited": false, + "label": "srpm", + "priority": 19, + "channel_id": 1, + "waiting": null, + "create_time": "2015-02-18 14:45:51.418503", + "id": 8982174, + "create_ts": 1424270751.4185, + "owner": 2045, + "host_id": 116, + "completion_ts": 1424271013.11918, + "arch": "noarch", + "method": "buildSRPMFromSCM" + } + }, + { + "method": "getHost", + "args": [ + 116 + ], + "kwargs": {}, + "result": { + "comment": null, + "capacity": 2.0, + "name": "arm02-builder13.build.example.com", + "enabled": false, + "arches": "armhfp", + "task_load": 0.0, + "ready": false, + "user_id": 2477, + "id": 116, + "description": null + } + }, + { + "method": "getTaskResult", + "args": [ + 8982174 + ], + "kwargs": {}, + "result": { + "brootid": 2978259, + "srpm": "tasks/2174/8982174/sisu-0.3.0-0.2.M1.fc23.src.rpm", + "logs": [ + "tasks/2174/8982174/build.log", + "tasks/2174/8982174/root.log", + "tasks/2174/8982174/state.log" + ] + } + }, + { + "method": "listTaskOutput", + "args": [ + 8982174 + ], + "kwargs": { + "all_volumes": true + }, + "result": {} + }, + { + "method": "getTaskRequest", + "args": [ + 8982174 + ], + "kwargs": {}, + "result": [ + "git://pkgs.example.com/sisu?#5428a6665ed5d4920bda5da6db6c9fce942070ac", + 299, + { + "repo_id": 456563 + } + ] + }, + { + "method": "getTaskChildren", + "args": [ + 8982174 + ], + "kwargs": {}, + "result": [] + }, + { + "method": "getTaskInfo", + "args": [ + 8982231 + ], + "kwargs": {}, + "result": { + "weight": 1.62572677375, + "parent": 8982172, + "completion_time": "2015-02-18 14:57:33.063121", + "start_time": "2015-02-18 14:50:43.011525", + "start_ts": 1424271043.01152, + "state": 2, + "awaited": false, + "label": "noarch", + "priority": 19, + "channel_id": 1, + "waiting": null, + "create_time": "2015-02-18 14:50:37.546755", + "id": 8982231, + "create_ts": 1424271037.54676, + "owner": 2045, + "host_id": 81, + "completion_ts": 1424271453.06312, + "arch": "noarch", + "method": "buildArch" + } + }, + { + "method": "getHost", + "args": [ + 81 + ], + "kwargs": {}, + "result": { + "comment": null, + "capacity": 4.0, + "name": "buildvm-23.build.example.com", + "enabled": true, + "arches": "i386 x86_64", + "task_load": 0.0, + "ready": true, + "user_id": 2176, + "id": 81, + "description": null + } + }, + { + "method": "getTaskResult", + "args": [ + 8982231 + ], + "kwargs": {}, + "result": { + "brootid": 2978278, + "srpms": [ + "tasks/2231/8982231/sisu-0.3.0-0.2.M1.fc23.src.rpm" + ], + "rpms": [ + "tasks/2231/8982231/sisu-javadoc-0.3.0-0.2.M1.fc23.noarch.rpm", + "tasks/2231/8982231/sisu-plexus-0.3.0-0.2.M1.fc23.noarch.rpm", + "tasks/2231/8982231/sisu-inject-0.3.0-0.2.M1.fc23.noarch.rpm" + ], + "logs": [ + "tasks/2231/8982231/root.log", + "tasks/2231/8982231/state.log", + "tasks/2231/8982231/build.log" + ] + } + }, + { + "method": "listTaskOutput", + "args": [ + 8982231 + ], + "kwargs": { + "all_volumes": true + }, + "result": {} + }, + { + "method": "getTaskRequest", + "args": [ + 8982231 + ], + "kwargs": {}, + "result": [ + "tasks/2174/8982174/sisu-0.3.0-0.2.M1.fc23.src.rpm", + 299, + "noarch", + true, + { + "repo_id": 456563 + } + ] + }, + { + "method": "getTaskChildren", + "args": [ + 8982231 + ], + "kwargs": {}, + "result": [] + }, + { + "method": "getChangelogEntries", + "args": [ + 612609 + ], + "kwargs": { + "queryOpts": { + "limit": 3 + } + }, + "result": [ + { + "date": "2015-02-18 12:00:00", + "text": "- Unbundle ASM\n- Resolves: rhbz#1085903", + "date_ts": 1424260800, + "author": "Happy Koji User - 1:0.3.0-0.2.M1" + }, + { + "date": "2015-02-04 12:00:00", + "text": "- Update to upstream milestone 0.3.0.M1", + "date_ts": 1423051200, + "author": "Happy Koji User - 1:0.3.0-0.1.M1" + }, + { + "date": "2014-09-30 12:00:00", + "text": "- Port to plexus-utils 3.0.18", + "date_ts": 1412078400, + "author": "Happy Koji User - 1:0.2.1-10" + } + ] + } +] diff --git a/tests/test_builder/data/calls/build_notif_1/message.txt b/tests/test_builder/data/calls/build_notif_1/message.txt new file mode 100644 index 0000000..34d3d1f --- /dev/null +++ b/tests/test_builder/data/calls/build_notif_1/message.txt @@ -0,0 +1,44 @@ +From: koji@example.com +Subject: Package: sisu-0.3.0-0.2.M1.fc23 Tag: f23 Status: complete Built by: user +To: user@example.com +X-Koji-Tag: f23 +X-Koji-Package: sisu +X-Koji-Builder: user +X-Koji-Status: complete + +Package: sisu-0.3.0-0.2.M1.fc23 +Tag: f23 +Status: complete +Built by: user +ID: 612609 +Started: Wed, 18 Feb 2015 14:50:37 EST +Finished: Wed, 18 Feb 2015 14:57:37 EST +Changelog: +* Wed Feb 18 2015 Happy Koji User - 1:0.3.0-0.2.M1 +- Unbundle ASM +- Resolves: rhbz#1085903 + +* Wed Feb 04 2015 Happy Koji User - 1:0.3.0-0.1.M1 +- Update to upstream milestone 0.3.0.M1 + +* Tue Sep 30 2014 Happy Koji User - 1:0.2.1-10 +- Port to plexus-utils 3.0.18 + + + +Closed tasks: +------------- + +Task 8982172 on buildvm-26.build.example.com +Task Type: build (rawhide, /sisu:5428a6665ed5d4920bda5da6db6c9fce942070ac) + +Task 8982174 on arm02-builder13.build.example.com +Task Type: buildSRPMFromSCM (/sisu:5428a6665ed5d4920bda5da6db6c9fce942070ac) + +Task 8982231 on buildvm-23.build.example.com +Task Type: buildArch (sisu-0.3.0-0.2.M1.fc23.src.rpm, noarch) + + + +Task Info: https://koji.example.com/kojiweb/taskinfo?taskID=8982172 +Build Info: https://koji.example.com/kojiweb/buildinfo?buildID=612609 diff --git a/tests/test_builder/data/calls/build_notif_1/params.json b/tests/test_builder/data/calls/build_notif_1/params.json new file mode 100644 index 0000000..64281ee --- /dev/null +++ b/tests/test_builder/data/calls/build_notif_1/params.json @@ -0,0 +1,35 @@ +{ + "recipients": [ + "user@example.com" + ], + "weburl": "https://koji.example.com/kojiweb", + "build": { + "task_id": 8982172, + "owner_name": "user", + "name": "sisu", + "package_id": 11059, + "state": 1, + "package_name": "sisu", + "creation_event_id": 9798602, + "creation_time": "2015-02-18 14:50:37.471074", + "owner_id": 2045, + "completion_time": "2015-02-18 14:57:37.107872", + "epoch": 1, + "version": "0.3.0", + "volume_id": 0, + "release": "0.2.M1.fc23", + "creation_ts": 1424271037.47107, + "completion_ts": 1424271457.10787, + "id": 612609, + "volume_name": "DEFAULT", + "nvr": "sisu-0.3.0-0.2.M1.fc23" + }, + "target": { + "dest_tag": 292, + "build_tag": 299, + "build_tag_name": "f23-build", + "dest_tag_name": "f23", + "id": 88, + "name": "rawhide" + } +} diff --git a/tests/test_builder/test_build_notification.py b/tests/test_builder/test_build_notification.py new file mode 100644 index 0000000..8f3e4a5 --- /dev/null +++ b/tests/test_builder/test_build_notification.py @@ -0,0 +1,94 @@ +from __future__ import absolute_import +import json +import mock +import os +import smtplib +import tempfile +import unittest + +import koji +import koji.util +from .loadkojid import kojid + + +class MyClientSession(koji.ClientSession): + + def __init__(self, *a, **kw): + super(MyClientSession, self).__init__(*a, **kw) + self._testcalls = {} + + def load_calls(self, name): + fn = os.path.join(os.path.dirname(__file__), 'data/calls', name,'calls.json') + with open(fn) as fp: + data = json.load(fp) + data = koji.fixEncodingRecurse(data) + for call in data: + key = self._munge([call['method'], call['args'], call['kwargs']]) + self._testcalls[key] = call + + def _callMethod(self, name, args, kwargs=None, retry=True): + if self.multicall: + raise Exception('multicall not supported') + key = self._munge([name, args, kwargs]) + if key in self._testcalls: + return self._testcalls[key]['result'] + else: + return mock.MagicMock() + + def _munge(self, data): + def callback(value): + if isinstance(value, list): + return tuple(value) + elif isinstance(value, dict): + keys = sorted(value.keys()) + return tuple([(k, value[k]) for k in keys]) + else: + return value + walker = koji.util.DataWalker(data, callback) + return walker.walk() + + +class TestBuildNotification(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.SMTP = mock.patch('smtplib.SMTP').start() + self.session = mock.MagicMock() + self.options = mock.MagicMock() + self.options.topdir = self.tempdir + self.options.workdir = self.tempdir + + def tearDown(self): + mock.patch.stopall() + + def test_build_notification(self): + # task_info['id'], method, params, self.session, self.options + task_id = 999 + fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'params.json') + with file(fn) as fp: + kwargs = json.load(fp) + self.session = MyClientSession('https://koji.example.com/kojihub') + self.session.load_calls('build_notif_1') + self.options.from_addr = "koji@example.com" + server = mock.MagicMock() + self.SMTP.return_value = server + + # run it + handler = kojid.BuildNotificationTask( + task_id, + 'buildNotification', + koji.encode_args(**kwargs), + self.session, + self.options) + ret = handler.run() + + self.assertEqual(ret, "sent notification of build 612609 to: user@example.com") + + # check sendmail args + from_addr, recipients, message = server.sendmail.call_args[0] + self.assertEqual(from_addr, "koji@example.com") + self.assertEqual(recipients, ["user@example.com"]) + fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'message.txt') + with file(fn) as fp: + msg_expect = fp.read() + self.assertEqual(message, msg_expect)