From c2f44491abbcfb9e290170ebf6a61faec34eb547 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Nov 02 2017 00:01:32 +0000 Subject: [PATCH 1/2] Add scenario values to the API response. At the request of @adamwill. Without this, it would be hard to understand responses about failed rawhide composes. --- diff --git a/functional-tests/consumers/test_resultsdb.py b/functional-tests/consumers/test_resultsdb.py index c0099ee..7816764 100644 --- a/functional-tests/consumers/test_resultsdb.py +++ b/functional-tests/consumers/test_resultsdb.py @@ -70,7 +70,8 @@ def test_consume_new_result( 'item': nvr, 'type': 'koji_build' }, - 'type': 'test-result-missing' + 'type': 'test-result-missing', + 'scenario': None, }, { 'testcase': 'dist.upgradepath', @@ -78,7 +79,8 @@ def test_consume_new_result( 'item': nvr, 'type': 'koji_build' }, - 'type': 'test-result-missing' + 'type': 'test-result-missing', + 'scenario': None, } ], 'summary': '2 of 3 required tests not found', diff --git a/functional-tests/test_api_v1.py b/functional-tests/test_api_v1.py index df70b27..a62be29 100644 --- a/functional-tests/test_api_v1.py +++ b/functional-tests/test_api_v1.py @@ -257,13 +257,15 @@ def test_make_a_decison_on_failed_result(requests_session, greenwave_server, tes 'item': {'item': nvr, 'type': 'koji_build'}, 'result_id': result['id'], 'testcase': 'dist.rpmdiff.comparison.xml_validity', + 'scenario': None, 'type': 'test-result-failed' }, ] + [ { 'item': {'item': nvr, 'type': 'koji_build'}, 'testcase': name, - 'type': 'test-result-missing' + 'type': 'test-result-missing', + 'scenario': None, } for name in all_rpmdiff_testcase_names if name != 'dist.rpmdiff.comparison.xml_validity' ] assert sorted(res_data['unsatisfied_requirements']) == sorted(expected_unsatisfied_requirements) @@ -289,7 +291,8 @@ def test_make_a_decison_on_no_results(requests_session, greenwave_server, testda { 'item': {'item': nvr, 'type': 'koji_build'}, 'testcase': name, - 'type': 'test-result-missing' + 'type': 'test-result-missing', + 'scenario': None, } for name in all_rpmdiff_testcase_names ] assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements @@ -379,7 +382,8 @@ def test_multiple_results_in_a_subject( 'item': {'item': nvr, 'type': 'koji_build'}, 'result_id': result['id'], 'testcase': 'dist.abicheck', - 'type': 'test-result-failed' + 'type': 'test-result-failed', + 'scenario': None, }, ] assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements @@ -420,7 +424,8 @@ def test_ignore_result(requests_session, greenwave_server, testdatabuilder): { 'item': {'item': nvr, 'type': 'koji_build'}, 'testcase': TASKTRON_RELEASE_CRITICAL_TASKS[0], - 'type': 'test-result-missing' + 'type': 'test-result-missing', + 'scenario': None, }, ] assert r.status_code == 200 @@ -530,7 +535,8 @@ def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder): 'item': {'item': nvr, 'type': 'koji_build'}, 'result_id': result['id'], 'testcase': all_rpmdiff_testcase_names[0], - 'type': 'test-result-failed' + 'type': 'test-result-failed', + 'scenario': None, }, ] assert res_data['policies_satisfied'] is False diff --git a/greenwave/policies.py b/greenwave/policies.py index f56d60d..f90ebb6 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -47,15 +47,17 @@ class TestResultMissing(RuleNotSatisfied): ResultsDB with a matching item and test case name). """ - def __init__(self, item, test_case_name): + def __init__(self, item, test_case_name, scenario): self.item = item self.test_case_name = test_case_name + self.scenario = scenario def to_json(self): return { 'type': 'test-result-missing', 'item': self.item, 'testcase': self.test_case_name, + 'scenario': self.scenario, } @@ -65,9 +67,10 @@ class TestResultFailed(RuleNotSatisfied): not ``PASSED`` or ``INFO``) and no corresponding waiver was found. """ - def __init__(self, item, test_case_name, result_id): + def __init__(self, item, test_case_name, scenario, result_id): self.item = item self.test_case_name = test_case_name + self.scenario = scenario self.result_id = result_id def to_json(self): @@ -75,6 +78,7 @@ class TestResultFailed(RuleNotSatisfied): 'type': 'test-result-failed', 'item': self.item, 'testcase': self.test_case_name, + 'scenario': self.scenario, 'result_id': self.result_id, } @@ -154,7 +158,7 @@ class PassingTestCaseRule(Rule): r['data'].get('scenario', [])] if not matching_results: - return TestResultMissing(item, self.test_case_name) + return TestResultMissing(item, self.test_case_name, self._scenario) # If we find multiple matching results, we always use the first one which # will be the latest chronologically, because ResultsDB always returns # results ordered by `submit_time` descending. @@ -164,7 +168,7 @@ class PassingTestCaseRule(Rule): # XXX limit who is allowed to waive if any(w['result_id'] == matching_result['id'] and w['waived'] for w in waivers): return RuleSatisfied() - return TestResultFailed(item, self.test_case_name, matching_result['id']) + return TestResultFailed(item, self.test_case_name, self._scenario, matching_result['id']) @property def _scenario(self): diff --git a/greenwave/tests/test_policies.py b/greenwave/tests/test_policies.py index c2d8d7e..c2fb390 100644 --- a/greenwave/tests/test_policies.py +++ b/greenwave/tests/test_policies.py @@ -13,14 +13,14 @@ from greenwave.utils import load_policies def test_summarize_answers(): assert summarize_answers([RuleSatisfied()]) == \ 'all required tests passed' - assert summarize_answers([TestResultFailed('item', 'test', 'id'), RuleSatisfied()]) == \ + assert summarize_answers([TestResultFailed('item', 'test', None, 'id'), RuleSatisfied()]) == \ '1 of 2 required tests failed' - assert summarize_answers([TestResultMissing('item', 'test')]) == \ + assert summarize_answers([TestResultMissing('item', 'test', None)]) == \ 'no test results found' - assert summarize_answers([TestResultMissing('item', 'test'), - TestResultFailed('item', 'test', 'id')]) == \ + assert summarize_answers([TestResultMissing('item', 'test', None), + TestResultFailed('item', 'test', None, 'id')]) == \ '1 of 2 required tests failed' - assert summarize_answers([TestResultMissing('item', 'test'), RuleSatisfied()]) == \ + assert summarize_answers([TestResultMissing('item', 'test', None), RuleSatisfied()]) == \ '1 of 2 required tests not found' From 3bca2f2defb57c15d9907f6bc358b64e52fa2d44 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Nov 02 2017 02:16:34 +0000 Subject: [PATCH 2/2] Ensure at least one test returns a non-None scenario in the response. --- diff --git a/functional-tests/test_api_v1.py b/functional-tests/test_api_v1.py index a62be29..191c421 100644 --- a/functional-tests/test_api_v1.py +++ b/functional-tests/test_api_v1.py @@ -83,9 +83,6 @@ TASKTRON_RELEASE_CRITICAL_TASKS = [ 'dist.upgradepath', ] -OPENQA_TASKS = [ - 'compose.install_no_user', -] OPENQA_SCENARIOS = [ 'scenario1', 'scenario2', @@ -440,12 +437,13 @@ def test_make_a_decison_on_passed_result_with_scenario( If we require two scenarios to pass, and both pass, then we pass. """ compose_id = testdatabuilder.unique_compose_id() - for testcase_name in OPENQA_TASKS: - for scenario in OPENQA_SCENARIOS: - testdatabuilder.create_result(item=compose_id, - testcase_name=testcase_name, - scenario=scenario, - outcome='PASSED') + testcase_name = 'compose.install_no_user' + for scenario in OPENQA_SCENARIOS: + testdatabuilder.create_result( + item=compose_id, + testcase_name=testcase_name, + scenario=scenario, + outcome='PASSED') data = { 'decision_context': 'rawhide_compose_sync_to_mirrors', 'product_version': 'fedora-rawhide', @@ -469,17 +467,19 @@ def test_make_a_decison_on_failing_result_with_scenario( """ compose_id = testdatabuilder.unique_compose_id() - for testcase_name in OPENQA_TASKS: - # Scenario 1 passes.. - testdatabuilder.create_result(item=compose_id, - testcase_name=testcase_name, - scenario='scenario1', - outcome='PASSED') - # But scenario 2 fails! - testdatabuilder.create_result(item=compose_id, - testcase_name=testcase_name, - scenario='scenario2', - outcome='FAILED') + testcase_name = 'compose.install_no_user' + # Scenario 1 passes.. + testdatabuilder.create_result( + item=compose_id, + testcase_name=testcase_name, + scenario='scenario1', + outcome='PASSED') + # But scenario 2 fails! + result = testdatabuilder.create_result( + item=compose_id, + testcase_name=testcase_name, + scenario='scenario2', + outcome='FAILED') data = { 'decision_context': 'rawhide_compose_sync_to_mirrors', 'product_version': 'fedora-rawhide', @@ -494,6 +494,14 @@ def test_make_a_decison_on_failing_result_with_scenario( assert res_data['applicable_policies'] == ['openqa_important_stuff_for_rawhide'] expected_summary = '1 of 2 required tests failed' assert res_data['summary'] == expected_summary + expected_unsatisfied_requirements = [{ + u'item': {u'item': compose_id}, + u'result_id': result['id'], + u'testcase': testcase_name, + u'type': u'test-result-failed', + u'scenario': u'scenario2', + }] + assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder):