From 8a56f5aa38c9b02f112f8115d4e9a4653e25410b Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Apr 25 2018 10:44:18 +0000 Subject: [PATCH 1/3] Update gitignore nad requirements.txt --- diff --git a/.gitignore b/.gitignore index 28f1a4c..b6dac6a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ # pytest cache /testing/.cache +.pytest_cache # virtualenv /env_trigger/ diff --git a/requirements.txt b/requirements.txt index 7d87b34..e1aa418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ fedmsg[consumers] mongoquery requests Twisted +koji PyYAML >= 3.11 # Test suite requirements From 2862aafd0676b275d939a3b5888e587d61ae17a3 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Apr 25 2018 10:44:24 +0000 Subject: [PATCH 2/3] Ondemand task scheduling Enables consumption of a FedMSG with `item`, `item_type`, `testcase` and `arch` fields, based on which (plus trigger configuration) a task responsible for creating/submitting results with that specific testcase/item/type is executed. --- diff --git a/conf/trigger_rules.yml.example b/conf/trigger_rules.yml.example index c11b09f..62821b8 100644 --- a/conf/trigger_rules.yml.example +++ b/conf/trigger_rules.yml.example @@ -63,3 +63,9 @@ message_type: GitHubPullRequestOpened do: - {tasks: [task-mtf]} + +- when: + message_type: TaskotronResultMissing + testcase: {$regex: '/^dist.rpmgrill(\..+)?/'} + do: + - {tasks: [rpmgrill], arches: ${arches}} diff --git a/docs/ondemand-task-scheduling.rst b/docs/ondemand-task-scheduling.rst new file mode 100644 index 0000000..cf768fa --- /dev/null +++ b/docs/ondemand-task-scheduling.rst @@ -0,0 +1,54 @@ +Taskotron-trigger ondemand task scheduling +========================================== + +With gating, we need to have a way of scheduling tasks for missing results. +A result can be missing for a bunch of reasons, from networking errors, to +unexpected system crashes/downtimes. + +The gating system required that each item under testing (from now on, let's +call it a build), passes a predefined set of criteria. +When a build is stuck in the gating process, because some results are not +available (after a reasonable amount of time), there needs to be a way of +running the task responsible for providing those results [again]. + +To facilitate this, we added a re-run functinality to the Taskotron-trigger. +As with all the stuff the Trigger does, it is based on an incoming FedMSG. +The required fields for the data enclosed in such message are ``item``, +``item_type``, ``testcase``. +We also allow for ``arch`` to be specified (as there might be a case when +results are missing only for a certain architecture). + +An example message could look like this:: + + {"i": 1, + "msg": {"item": "net-tools-2.0-0.45.20160912git.fc26", + "item_type": "koji_build", + "testcase": "dist.rpmgrill.desktop-lint", + "arch": "x86_64"}, + "timestamp": 1359603469.21164, + "topic": "org.fedoraproject.prod.taskotron.result.missing", + "username": "coconut"}' + +We do not necessarily require this precise format (even though the code +as isworks with that in mind), but all the data must be present in order +for us to be able to run the appropriate task. +If the message structure, field names, or topic changes, we basically do +not care, as long as all the required data is there in an anticipated place. + + +Future implementation discussion +-------------------------------- + +The aforementioned (and currently implemented) method is very crude, +and prone to all kinds of kinks. It is the baseline, on which we could +build a more robust solution. This could encompass steps like looking into +the ExecDB to check whether a job that could provide such result was already +executed/scheduled, or managing the communication back to the "requestor" - +quite probably by emitting a fedmsg saying "yeah, we ran X in response to the +message Y", or marking such jobs respectively in ExecDB for further audit. + +We should also make sure the re-run requests are rate-limited at least to +some extent so we don't end up DoSing the Taskotron stack. + +But this is most definitely on top of the required baseline, and we can assume +"good behaviour" (or at least "good intent") for the start. diff --git a/fedmsg.d/taskotron-trigger.py b/fedmsg.d/taskotron-trigger.py index 8c2afd0..6d2f77a 100644 --- a/fedmsg.d/taskotron-trigger.py +++ b/fedmsg.d/taskotron-trigger.py @@ -6,4 +6,5 @@ config = { 'taskotron.cloudcomposecompletedconsumer.enabled': True, 'taskotron.modulebuilddonejobconsumer.enabled': True, 'taskotron.githubpropenedjobconsumer.enabled': True, + 'taskotron.taskotronresultmissingjobconsumer.enabled': True, } diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py index 5cfc1c1..389026d 100644 --- a/jobtriggers/jobtrigger.py +++ b/jobtriggers/jobtrigger.py @@ -92,6 +92,9 @@ class JobTrigger(object): else: self.log.error("Rule has invalid `do` section: %r", rule) + if 'arches' in item: + task['arches'] = item['arches'] + tasks.append(task) return tasks diff --git a/jobtriggers/mongoquery_string_template.py b/jobtriggers/mongoquery_string_template.py index dc7a6d5..4b6e41b 100644 --- a/jobtriggers/mongoquery_string_template.py +++ b/jobtriggers/mongoquery_string_template.py @@ -79,6 +79,12 @@ class MongoTemplate(string.Template): val = mapping[named] # We use this idiom instead of str() because the latter will # fail if val is a Unicode containing non-ASCII characters. + # FIXME - Add coverage for the list/dict handling + if isinstance(val, list) or isinstance(val, tuple): + return "[%s]" % ",".join(val) + if isinstance(val, dict): + items = ["%s: %s" % (k, v) for k, v in val.iteritems()] + return "{%s}" % ",".join(items) return '%s' % (val,) if mo.group('escaped') is not None: return self.delimiter @@ -109,9 +115,16 @@ class MongoTemplate(string.Template): named = mo.group('named') or mo.group('braced') if named is not None: try: + val = mapping[named] # We use this idiom instead of str() because the latter # will fail if val is a Unicode containing non-ASCII - return '%s' % (mapping[named],) + # FIXME - Add coverage for the list/dict handling + if isinstance(val, list) or isinstance(val, tuple): + return "[%s]" % ",".join(val) + if isinstance(val, dict): + items = ["%s: %s" % (k, v) for k, v in val.iteritems()] + return "{%s}" % ",".join(items) + return '%s' % (val,) except KeyError: return mo.group() if mo.group('escaped') is not None: diff --git a/jobtriggers/taskotron_result_missing_msg.py b/jobtriggers/taskotron_result_missing_msg.py new file mode 100644 index 0000000..286c88d --- /dev/null +++ b/jobtriggers/taskotron_result_missing_msg.py @@ -0,0 +1,45 @@ +import fedmsg.consumers + +from . import config +from .jobtrigger import JobTrigger +from . import exceptions as exc + +MESSAGE_TYPE = 'TaskotronResultMissing' + +class TaskotronResultMissingJobTrigger(JobTrigger): + def process(self, msg): + item = msg['msg']['item'] + item_type = msg['msg']['item_type'] + testcase = msg['msg']['testcase'] + if 'arch' in msg['msg']: + arches = [msg['msg']['arch']] + else: + arches = config.valid_arches + + data = { + "_msg": msg, + "message_type": MESSAGE_TYPE, + "item": item, + "item_type": item_type, + "testcase": testcase, + "arches": arches, + } + return data + + +class TasktoronResultMissingJobConsumer(fedmsg.consumers.FedmsgConsumer): + topic = "org.fedoraproject.%s.taskotron.result.missing" % config.deployment_type + config_key = 'taskotron.taskotronresultmissingjobconsumer.enabled' + + def __init__(self, *args, **kw): + super(TasktoronResultMissingJobConsumer, self).__init__(*args, **kw) + self.trigger = TaskotronResultMissingJobTrigger(self.log) + + def consume(self, message): + msg = fedmsg.encoding.loads(message.body) + try: + data = self.trigger.process(msg) + self.trigger.do_trigger(data) + except exc.TriggerMsgError, e: + self.log.debug(e) + return diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py index 9fa5921..6b3ed5f 100644 --- a/testing/test_jobtrigger.py +++ b/testing/test_jobtrigger.py @@ -81,6 +81,17 @@ class TestJobtrigger(): assert tasks[0]['item_type'] == self.ref_message_data['item_type'] assert tasks[0]['arches'] == self.ref_default_arches + + def test__tasks_from_rule_with_arches(self): + rule = {'do': [{'tasks': self.ref_tasks, 'arches': ['bogusarch']}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert len(tasks) == 1 + assert tasks[0]['tasks'] == self.ref_tasks + assert tasks[0]['item'] == self.ref_message_data['item'] + assert tasks[0]['item_type'] == self.ref_message_data['item_type'] + assert tasks[0]['arches'] == ['bogusarch'] + + def test__tasks_from_rule_discover(self): ref_repo = 'http://bogus.repo/firefox.git' rule = {'do': [{'discover': {'repo': ref_repo}}]} @@ -140,6 +151,34 @@ class TestJobtrigger(): assert runner_calls[3][1] == ( self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1]) + def test_do_trigger_runtasks_with_arches(self, monkeypatch): + ref_arches = ['arch_foo', 'arch_bar'] + mock__load_rules = mock.Mock(return_value=[ + { + 'do': [{'tasks': self.ref_tasks, 'arches': ref_arches}], + 'when': {'message_type': 'RunTasks'} + }, + ]) + monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules) + + message_data = copy.deepcopy(self.ref_message_data) + message_data['message_type'] = 'RunTasks' + + self.helper.do_trigger(message_data) + + runner_calls = self.helper.runner.trigger_job.calls() + + assert len(runner_calls) == 4 + + assert runner_calls[0][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0]) + assert runner_calls[1][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1]) + assert runner_calls[2][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0]) + assert runner_calls[3][1] == ( + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1]) + def test_do_trigger_discover(self, monkeypatch): mock__load_rules = mock.Mock(return_value=self.ref_rules) monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules) diff --git a/testing/test_taskotron_result_missing_trigger.py b/testing/test_taskotron_result_missing_trigger.py new file mode 100644 index 0000000..6461e66 --- /dev/null +++ b/testing/test_taskotron_result_missing_trigger.py @@ -0,0 +1,68 @@ +import pytest +from dingus import Dingus +from munch import Munch +from copy import deepcopy +import fedmsg.encoding +import fedmsg.consumers + +from jobtriggers import taskotron_result_missing_msg + + +@pytest.mark.usefixtures('prepare') +class TestTaskotronResultMissingJobConsumer(): + + @pytest.fixture + def prepare(self, monkeypatch): + self.ref_item = 'ref_item' + self.ref_item_type = 'ref_item_type' + self.ref_testcase = 'ref.testcase.subcase' + self.ref_arch = 'ref_arch' + self.ref_task = 'ref_task' + self.ref_validarches = ['i386', 'x86_64'] + + self._create_msg( + self.ref_item, self.ref_item_type, self.ref_testcase, self.ref_arch) + + self.ref_data = { + "_msg": {}, + "message_type": "TaskotronResultMissing", + "item": self.ref_item, + "item_type": self.ref_item_type, + "testcase": self.ref_testcase, + "arches": [self.ref_arch], + } + + stub_hub = Munch(config=Munch(get=0)) + self.helper = taskotron_result_missing_msg.TasktoronResultMissingJobConsumer(stub_hub) + + self.helper.trigger.runner = Dingus() + taskotron_result_missing_msg.config.trigger_rules_template = """--- +- do: + - {tasks: [ref_task], arches: ${arches}} + when: {message_type: TaskotronResultMissing, testcase: {$regex: '/^ref.testcase(\..+)*/'}} +""" + + taskotron_result_missing_msg.config.valid_arches = self.ref_validarches + taskotron_result_missing_msg.config.job_logging = False + + def _create_msg(self, ref_item, ref_item_type, ref_testcase, ref_arch): + self.ref_message = Munch(body='{"i": 1,\ + "msg": {"item": "%s",\ + "item_type": "%s",\ + "testcase": "%s",\ + "arch": "%s"},\ + "timestamp": 1359603469.21164,\ + "topic": "org.fedoraproject.prod.taskotron.result.missing",\ + "username": "apache"}' % + (ref_item, ref_item_type, ref_testcase, ref_arch)) + + def test_consume(self): + self.helper.consume(self.ref_message) + + runner_calls = self.helper.trigger.runner.trigger_job.calls() + + assert len(runner_calls) == 1 + + assert runner_calls[0][1] == (self.ref_item, self.ref_item_type, + self.ref_task, self.ref_arch) + From d01478c45175bdc605dee66e80fe20b5ad924145 Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Apr 25 2018 11:03:48 +0000 Subject: [PATCH 3/3] Make fixed-architecture job scheduling possible --- diff --git a/conf/trigger.cfg.example b/conf/trigger.cfg.example index 553245d..355f95e 100644 --- a/conf/trigger.cfg.example +++ b/conf/trigger.cfg.example @@ -3,6 +3,7 @@ url = http://localhost:8080/buildbot/change_hook [trigger] valid_arches = x86_64 +default_run_arch = noarch datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw execdb_server = http://localhost:5003 runner_type = StreamRunner diff --git a/jobtriggers/config.py b/jobtriggers/config.py index 91bbc69..6694405 100644 --- a/jobtriggers/config.py +++ b/jobtriggers/config.py @@ -7,6 +7,7 @@ url = http://localhost:8080/buildbot/change_hook [trigger] valid_arches = x86_64 +default_run_arch = noarch datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw execdb_server = http://localhost:5003 runner_type = StreamRunner @@ -29,6 +30,7 @@ config.read(['/etc/taskotron/trigger.cfg', './conf/trigger.cfg']) buildbot_url = config.get('buildbot', 'url') valid_arches = [e.strip() for e in config.get('trigger', 'valid_arches').split(',') if e] +default_run_arch = config.get('trigger', 'default_run_arch') datagrepper_url = config.get('trigger', 'datagrepper_url') execdb_server = config.get('trigger', 'execdb_server') runner_type = config.get('trigger', 'runner_type') diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py index 389026d..161b922 100644 --- a/jobtriggers/jobtrigger.py +++ b/jobtriggers/jobtrigger.py @@ -26,23 +26,24 @@ class JobTrigger(object): self.log = log self.runner = runners.get_runner(config.runner_type) - def trigger_tasks(self, item, item_type, tasks, arches, **kwargs): + def trigger_jobs(self, item, item_type, tasks, arches, arch_dependent, **kwargs): for task, arch in itertools.product(tasks, arches): - self.log.info('triggering %s for %s on %s', task, item, arch) + run_arch = arch if arch_dependent else config.default_run_arch # FIXME make 'noarch' configurable + self.log.info('triggering %s for %s on %s (run on %s)', task, item, arch, run_arch) if config.job_logging: try: - utils.log_job(config.joblog_file, item, item_type, task, arch) + utils.log_job(config.joblog_file, item, item_type, task, arch, run_arch) except IOError, e: self.log.exception(e) - output = self.runner.trigger_job(item, item_type, task, arch, **kwargs) + output = self.runner.trigger_job(item, item_type, task, arch, run_arch, **kwargs) for line in output: self.log.info(line) def _tasks_from_rule(self, rule, message_data): """Takes a rule-dict as input, and produces list of dicts representing arguments required - by the trigger_tasks method (i.e. item, item_type, tasks, arches) plus any additional data + by the trigger_jobs method (i.e. item, item_type, tasks, arches) plus any additional data provided in the rule. """ @@ -52,6 +53,7 @@ class JobTrigger(object): 'item': message_data['item'], 'item_type': message_data['item_type'], 'arches': config.valid_arches, + 'arch_dependent': False, 'tasks': [] } except KeyError, e: @@ -95,6 +97,9 @@ class JobTrigger(object): if 'arches' in item: task['arches'] = item['arches'] + if item.get('arch_dependent', False): + task['arch_dependent'] = True + tasks.append(task) return tasks @@ -133,7 +138,7 @@ class JobTrigger(object): for task in tasks: try: - self.trigger_tasks(**task) + self.trigger_jobs(**task) except exc.TriggerMsgError, e: self.log.debug(e) except exc.TriggerError, e: diff --git a/jobtriggers/runners.py b/jobtriggers/runners.py index 85ee02a..ea36b40 100644 --- a/jobtriggers/runners.py +++ b/jobtriggers/runners.py @@ -13,22 +13,24 @@ class BaseRunner(object): 'item': , 'item_type': , 'taskname': , - 'arch': + 'arch': , + 'run_arch': } """ - def trigger_job(self, item, item_type, taskname, arch, **kwargs): + def trigger_job(self, item, item_type, taskname, arch, run_arch, **kwargs): raise NotImplementedError class StreamRunner(BaseRunner): - def trigger_job(self, item, item_type, taskname, arch, **kwargs): + def trigger_job(self, item, item_type, taskname, arch, run_arch, **kwargs): output = ["StreamRunner received:", "item: %s" % item, "item_type: %s" % item_type, "taskname: %s" % taskname, - "arch: %s" % arch] + "arch: %s" % arch, + "run_arch: %s" % run_arch] output += ["%s: %s" % (k, w) for k, w in kwargs.iteritems()] yield '\n'.join(output) @@ -36,12 +38,13 @@ class StreamRunner(BaseRunner): class BuildbotRunner(BaseRunner): - def trigger_job(self, item, item_type, taskname, arch, **kwargs): - output = ["Buildbot received:", + def trigger_job(self, item, item_type, taskname, arch, run_arch, **kwargs): + output = ["BuildbotRunner received:", "item: %s" % item, "item_type: %s" % item_type, "taskname: %s" % taskname, - "arch: %s" % arch] + "arch: %s" % arch, + "run_arch: %s" % run_arch] yield '\n'.join(output) @@ -50,6 +53,7 @@ class BuildbotRunner(BaseRunner): 'item_type': item_type, 'taskname': taskname, 'arch': arch, + 'run_arch': run_arch, } properties.update(kwargs) @@ -82,7 +86,7 @@ class BuildbotRunner(BaseRunner): r = requests.post(config.buildbot_url, data={'author': 'taskotron', 'project': project, - 'category': arch, + 'category': run_arch, 'repository': '', 'comments': 'build request from taskotron-trigger', 'properties': json.dumps(properties)}) diff --git a/jobtriggers/utils.py b/jobtriggers/utils.py index d2e73a3..921ea45 100644 --- a/jobtriggers/utils.py +++ b/jobtriggers/utils.py @@ -13,12 +13,12 @@ from jobtriggers import config from jobtriggers import exceptions as exc -def log_job(logname, item, item_type, task, arch): +def log_job(logname, item, item_type, task, arch, arch_dependent): now = datetime.datetime.now() with open(logname, 'a') as csvfile: writer = csv.writer(csvfile, delimiter=';') - writer.writerow([now, item, item_type, task, arch]) + writer.writerow([now, item, item_type, task, arch, arch_dependent]) def parse_yaml_from_file(filename): diff --git a/testing/test_dist_git_trigger.py b/testing/test_dist_git_trigger.py index 04e45e1..328bc0a 100644 --- a/testing/test_dist_git_trigger.py +++ b/testing/test_dist_git_trigger.py @@ -46,6 +46,7 @@ class TestDistGitCommitJobConsumer(): monkeypatch.setattr(utils, 'parse_yaml_from_file', mock_get_critpath) dist_git_commit_msg.config.valid_arches = ['x86_64'] + dist_git_commit_msg.config.default_run_arch = 'noarch' def _create_msg(self): return Munch(body={ @@ -113,4 +114,5 @@ class TestDistGitCommitJobConsumer(): 'dist_git_commit', 'speclint', 'x86_64', + 'noarch', ) diff --git a/testing/test_github_pr_opened_trigger.py b/testing/test_github_pr_opened_trigger.py index 0bdfdaa..0871ed5 100644 --- a/testing/test_github_pr_opened_trigger.py +++ b/testing/test_github_pr_opened_trigger.py @@ -50,6 +50,7 @@ class TestKojiTagChangedJobConsumer(): github_pull_request_opened.config.valid_arches = self.ref_validarches github_pull_request_opened.config.job_logging = False + github_pull_request_opened.config.default_run_arch = 'noarch' def _create_msg(self, ref_pr_url, ref_repo_name): return Munch(body='{"i": 1,\ @@ -70,17 +71,17 @@ class TestKojiTagChangedJobConsumer(): assert len(runner_calls) == 6 assert runner_calls[0][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') assert runner_calls[4][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[5][1] == (self.ref_pr_url, self.ref_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') def test_trigger_data(self): trigger = github_pull_request_opened.GithubPullRequestOpenedJobTrigger(Dingus()) diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py index 6b3ed5f..7be4791 100644 --- a/testing/test_jobtrigger.py +++ b/testing/test_jobtrigger.py @@ -49,6 +49,7 @@ class TestJobtrigger(): ] jobtrigger.config.valid_arches = self.ref_default_arches + jobtrigger.config.default_run_arch = 'noarch' # short-circuit the logic to check if a repo exists stub_repoexist = mock.Mock(return_value=True) @@ -80,6 +81,7 @@ class TestJobtrigger(): assert tasks[0]['item'] == self.ref_message_data['item'] assert tasks[0]['item_type'] == self.ref_message_data['item_type'] assert tasks[0]['arches'] == self.ref_default_arches + assert tasks[0]['arch_dependent'] == False def test__tasks_from_rule_with_arches(self): @@ -90,6 +92,37 @@ class TestJobtrigger(): assert tasks[0]['item'] == self.ref_message_data['item'] assert tasks[0]['item_type'] == self.ref_message_data['item_type'] assert tasks[0]['arches'] == ['bogusarch'] + assert tasks[0]['arch_dependent'] == False + + def test__tasks_from_rule_with_arch_dependent(self): + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': True}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert len(tasks) == 1 + assert tasks[0]['tasks'] == self.ref_tasks + assert tasks[0]['item'] == self.ref_message_data['item'] + assert tasks[0]['item_type'] == self.ref_message_data['item_type'] + assert tasks[0]['arches'] == self.ref_default_arches + assert tasks[0]['arch_dependent'] == True + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': 'yes'}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == True + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': 'no'}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == True + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': False}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == False + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': 0}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == False + + rule = {'do': [{'tasks': self.ref_tasks, 'arch_dependent': ''}]} + tasks = self.helper._tasks_from_rule(rule, self.ref_message_data) + assert tasks[0]['arch_dependent'] == False def test__tasks_from_rule_discover(self): @@ -143,13 +176,13 @@ class TestJobtrigger(): assert len(runner_calls) == 4 assert runner_calls[0][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[0], 'noarch') assert runner_calls[1][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], self.ref_default_arches[1], 'noarch') assert runner_calls[2][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[0], 'noarch') assert runner_calls[3][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1], 'noarch') def test_do_trigger_runtasks_with_arches(self, monkeypatch): ref_arches = ['arch_foo', 'arch_bar'] @@ -171,13 +204,13 @@ class TestJobtrigger(): assert len(runner_calls) == 4 assert runner_calls[0][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0], 'noarch') assert runner_calls[1][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1], 'noarch') assert runner_calls[2][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0], 'noarch') assert runner_calls[3][1] == ( - self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1]) + self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1], 'noarch') def test_do_trigger_discover(self, monkeypatch): mock__load_rules = mock.Mock(return_value=self.ref_rules) @@ -194,10 +227,10 @@ class TestJobtrigger(): assert runner_calls[0][1] == ( self.ref_item, self.ref_item_type, self.ref_discovered_tasks[0], - self.ref_default_arches[0]) + self.ref_default_arches[0], 'noarch') assert runner_calls[1][1] == ( self.ref_item, self.ref_item_type, self.ref_discovered_tasks[0], - self.ref_default_arches[1]) + self.ref_default_arches[1], 'noarch') def test_do_trigger_disabled(self, monkeypatch): mock__load_rules = mock.Mock(return_value=self.ref_rules) diff --git a/testing/test_koji_build_trigger.py b/testing/test_koji_build_trigger.py index a8ed649..fd2dd45 100644 --- a/testing/test_koji_build_trigger.py +++ b/testing/test_koji_build_trigger.py @@ -55,6 +55,7 @@ class TestKojiBuildCompletedJobConsumer(): """ % (self.ref_tasks, self.message_type, self.ref_tasks_docker, self.message_type) koji_build_msg.config.valid_arches = self.ref_validarches + koji_build_msg.config.default_run_arch = 'noarch' koji_build_msg.config.job_logging = False mock_discover_tasks = mock.Mock(return_value=[]) @@ -101,13 +102,13 @@ class TestKojiBuildCompletedJobConsumer(): assert len(runner_calls) == 4 assert runner_calls[0][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') def test_consume_docker(self): # skip test when item is docker image @@ -124,17 +125,17 @@ class TestKojiBuildCompletedJobConsumer(): assert len(runner_calls) == 6 assert runner_calls[0][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') assert runner_calls[4][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks_docker[0], self.ref_validarches[0]) + self.ref_tasks_docker[0], self.ref_validarches[0], 'noarch') assert runner_calls[5][1] == (self.ref_nvr_docker, self.item_type, - self.ref_tasks_docker[0], self.ref_validarches[1]) + self.ref_tasks_docker[0], self.ref_validarches[1], 'noarch') def test_consume_distgit(self, monkeypatch): ref_distgit_task = 'abicheck' @@ -148,17 +149,17 @@ class TestKojiBuildCompletedJobConsumer(): assert len(runner_calls) == 6 assert runner_calls[0][1] == (self.ref_nvr, self.item_type, - ref_distgit_task, self.ref_validarches[0]) + ref_distgit_task, self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == (self.ref_nvr, self.item_type, - ref_distgit_task, self.ref_validarches[1]) + ref_distgit_task, self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[4][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[5][1] == (self.ref_nvr, self.item_type, - self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tasks[1], self.ref_validarches[1], 'noarch') def test_consume_not_primary_instance(self): ref_instance = 'ppc' diff --git a/testing/test_koji_tag_trigger.py b/testing/test_koji_tag_trigger.py index e0bddf3..dd6aab9 100644 --- a/testing/test_koji_tag_trigger.py +++ b/testing/test_koji_tag_trigger.py @@ -49,6 +49,7 @@ class TestKojiTagChangedJobConsumer(): """ koji_tag_msg.config.valid_arches = self.ref_validarches + koji_tag_msg.config.default_run_arch = 'noarch' koji_tag_msg.config.job_logging = False def _create_msg(self, ref_instance, ref_name, ref_release, ref_tag, ref_version): @@ -90,13 +91,13 @@ class TestKojiTagChangedJobConsumer(): assert len(runner_calls) == 4 assert runner_calls[0][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert runner_calls[2][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[0]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[0], 'noarch') assert runner_calls[3][1] == ( - self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[1]) + self.ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[1], self.ref_validarches[1], 'noarch') assert self.helper.queued_tags == set() assert self.helper.queued_data == dict() @@ -135,9 +136,9 @@ class TestKojiTagChangedJobConsumer(): assert len(runner_calls) == 2 assert runner_calls[0][1] == ( - ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0]) + ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[0], 'noarch') assert runner_calls[1][1] == ( - ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1]) + ref_tag, koji_tag_msg.ITEM_TYPE, self.ref_tasks[0], self.ref_validarches[1], 'noarch') assert self.helper.queued_tags == set() diff --git a/testing/test_module_build_trigger.py b/testing/test_module_build_trigger.py index 34b00e0..ac7c620 100644 --- a/testing/test_module_build_trigger.py +++ b/testing/test_module_build_trigger.py @@ -31,6 +31,7 @@ class TestModuleBuildDoneJobConsumer(): module_build_msg.config.job_logging = False module_build_msg.config.valid_arches = ['x86_64'] + module_build_msg.config.default_run_arch = 'noarch' def _create_msg(self, name, stream, version, state_name='done'): return Munch(body={ @@ -117,6 +118,7 @@ class TestModuleBuildDoneJobConsumer(): module_build_msg.ITEM_TYPE, self.ref_task, 'x86_64', + 'noarch', ) def test_consume_not_done(self): diff --git a/testing/test_pagure_commit_trigger.py b/testing/test_pagure_commit_trigger.py index 33bc68c..b97de46 100644 --- a/testing/test_pagure_commit_trigger.py +++ b/testing/test_pagure_commit_trigger.py @@ -43,6 +43,7 @@ class TestPagureGitReceivedJobConsumer(): monkeypatch.setattr(utils, 'clone_repo', mock_clone_repo) pagure_git_received.config.valid_arches = ['x86_64'] + pagure_git_received.config.default_run_arch = 'noarch' def _create_msg(self): return Munch(body={ @@ -115,4 +116,5 @@ class TestPagureGitReceivedJobConsumer(): 'git_commit', 'speclint', 'x86_64', + 'noarch', ) diff --git a/testing/test_taskotron_result_missing_trigger.py b/testing/test_taskotron_result_missing_trigger.py index 6461e66..ddf42f4 100644 --- a/testing/test_taskotron_result_missing_trigger.py +++ b/testing/test_taskotron_result_missing_trigger.py @@ -44,6 +44,7 @@ class TestTaskotronResultMissingJobConsumer(): taskotron_result_missing_msg.config.valid_arches = self.ref_validarches taskotron_result_missing_msg.config.job_logging = False + taskotron_result_missing_msg.config.default_run_arch = 'noarch' def _create_msg(self, ref_item, ref_item_type, ref_testcase, ref_arch): self.ref_message = Munch(body='{"i": 1,\ @@ -64,5 +65,5 @@ class TestTaskotronResultMissingJobConsumer(): assert len(runner_calls) == 1 assert runner_calls[0][1] == (self.ref_item, self.ref_item_type, - self.ref_task, self.ref_arch) + self.ref_task, self.ref_arch, 'noarch')