PR#96: Get "scenarios" working for greenwave. - greenwave

greenwave

#96 Get "scenarios" working for greenwave.

Merged 6 years ago by ralph. Opened 6 years ago by ralph.

with-scenarios into master

Get "scenarios" working for greenwave.

Ralph Bean • 6 years ago

0bdaae6

conf/policies/fedora.yaml

file modified

		`@@ -31,3 +31,12 @@`
		`rules:`
		`- !PassingTestCaseRule {test_case_name: dist.rpmdeplint}`
		`- !PassingTestCaseRule {test_case_name: dist.upgradepath}`
		`+ --- !Policy`
		`+ id: "openqa_important_stuff_for_rawhide"`
		`+ product_versions:`
		`+ - fedora-rawhide`
		`+ decision_context: rawhide_compose_sync_to_mirrors`
		`+ blacklist: []`
		`+ rules:`
		`+ - !PassingTestCaseRule {test_case_name: compose.install_no_user, scenario: scenario1}`
		`+ - !PassingTestCaseRule {test_case_name: compose.install_no_user, scenario: scenario2}`

functional-tests/conftest.py

file modified

+6 -1

		`@@ -123,12 +123,17 @@`
		`def unique_nvr(self):`
		`return 'glibc-1.0-{}.el7'.format(self._counter.next())`

		`- def create_result(self, item, testcase_name, outcome):`
		`+ def unique_compose_id(self):`
		`+ return 'Fedora-9000-19700101.n.{}'.format(self._counter.next())`
		`+`
		`+ def create_result(self, item, testcase_name, outcome, scenario=None):`
		`data = {`
		`'testcase': {'name': testcase_name},`
		`'data': {'item': item, 'type': 'koji_build'},`
		`'outcome': outcome,`
		`}`
		`+ if scenario:`
		`+ data['data']['scenario'] = scenario`
		`response = self.requests_session.post(`
		`self.resultsdb_url + 'api/v2.0/results',`
		`headers={'Content-Type': 'application/json'},`

functional-tests/test_api_v1.py

file modified

+75 -4

		`@@ -83,6 +83,14 @@`
		`'dist.upgradepath',`
		`]`

		`+ OPENQA_TASKS = [`
		`+ 'compose.install_no_user',`
		`+ ]`
		`+ OPENQA_SCENARIOS = [`
		`+ 'scenario1',`
		`+ 'scenario2',`
		`+ ]`
		`+`

		`def test_inspect_policies(requests_session, greenwave_server):`
		`r = requests_session.get(greenwave_server.url + 'api/v1.0/policies',`
		`@@ -90,20 +98,23 @@`
		`assert r.status_code == 200`
		`body = r.json()`
		`policies = body['policies']`
		`- assert len(policies) == 4`
		`+ assert len(policies) == 5`
		`assert any(p['id'] == 'taskotron_release_critical_tasks' for p in policies)`
		`assert any(p['decision_context'] == 'bodhi_update_push_stable' for p in policies)`
		`assert any(p['product_versions'] == ['fedora-26'] for p in policies)`
		`expected_rules = [`
		`{'rule': 'PassingTestCaseRule',`
		`- 'test_case_name': 'dist.abicheck'},`
		`+ 'test_case_name': 'dist.abicheck',`
		`+ 'scenario': None},`
		`]`
		`assert any(p['rules'] == expected_rules for p in policies)`
		`expected_rules = [`
		`{'rule': 'PassingTestCaseRule',`
		`- 'test_case_name': 'dist.rpmdeplint'},`
		`+ 'test_case_name': 'dist.rpmdeplint',`
		`+ 'scenario': None},`
		`{'rule': 'PassingTestCaseRule',`
		`- 'test_case_name': 'dist.upgradepath'}]`
		`+ 'test_case_name': 'dist.upgradepath',`
		`+ 'scenario': None}]`
		`assert any(p['rules'] == expected_rules for p in policies)`


		`@@ -418,6 +429,66 @@`
		`assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements`


		`+ def test_make_a_decison_on_passed_result_with_scenario(requests_session, greenwave_server, testdatabuilder):`
		`+ """`
		`+ If we require two scenarios to pass, and both pass, then we pass.`
		`+ """`
		`+ compose_id = testdatabuilder.unique_compose_id()`
		`+ for testcase_name in OPENQA_TASKS:`
		`+ for scenario in OPENQA_SCENARIOS:`
		`+ testdatabuilder.create_result(item=compose_id,`
		`+ testcase_name=testcase_name,`
		`+ scenario=scenario,`
		`+ outcome='PASSED')`
		`+ data = {`
		`+ 'decision_context': 'rawhide_compose_sync_to_mirrors',`
		`+ 'product_version': 'fedora-rawhide',`
		`+ 'subject': [{'item': compose_id}],`
		`+ }`
		`+ r = requests_session.post(greenwave_server.url + 'api/v1.0/decision',`
		`+ headers={'Content-Type': 'application/json'},`
		`+ data=json.dumps(data))`
		`+ assert r.status_code == 200`
		`+ res_data = r.json()`
		`+ assert res_data['policies_satisified'] is True`
		`+ assert res_data['applicable_policies'] == ['openqa_important_stuff_for_rawhide']`
		`+ expected_summary = 'all required tests passed'`
		`+ assert res_data['summary'] == expected_summary`
		`+`
		`+`
		`+ def test_make_a_decison_on_failing_result_with_scenario(requests_session, greenwave_server, testdatabuilder):`
		`+ """`
		`+ If we require two scenarios to pass, and one is failing, then we fail.`
		`+ """`
		`+`
		`+ compose_id = testdatabuilder.unique_compose_id()`
		`+ for testcase_name in OPENQA_TASKS:`
		`+ # Scenario 1 passes..`
		`+ testdatabuilder.create_result(item=compose_id,`
		`+ testcase_name=testcase_name,`
		`+ scenario='scenario1',`
		`+ outcome='PASSED')`
		`+ # But scenario 2 fails!`
		`+ testdatabuilder.create_result(item=compose_id,`
		`+ testcase_name=testcase_name,`
		`+ scenario='scenario2',`
		`+ outcome='FAILED')`
		`+ data = {`
		`+ 'decision_context': 'rawhide_compose_sync_to_mirrors',`
		`+ 'product_version': 'fedora-rawhide',`
		`+ 'subject': [{'item': compose_id}],`
		`+ }`
		`+ r = requests_session.post(greenwave_server.url + 'api/v1.0/decision',`
		`+ headers={'Content-Type': 'application/json'},`
		`+ data=json.dumps(data))`
		`+ assert r.status_code == 200`
		`+ res_data = r.json()`
		`+ assert res_data['policies_satisified'] is False`
		`+ assert res_data['applicable_policies'] == ['openqa_important_stuff_for_rawhide']`
		`+ expected_summary = '1 of 2 required tests failed'`
		`+ assert res_data['summary'] == expected_summary`
		`+`
		`+`
		`def test_ignore_waiver(requests_session, greenwave_server, testdatabuilder):`
		`"""`
		`This tests that a waiver can be ignored when making the decision.`

greenwave/policies.py

file modified

+11 -1

		`@@ -147,6 +147,11 @@`

		`def check(self, item, results, waivers):`
		`matching_results = [r for r in results if r['testcase']['name'] == self.test_case_name]`
		`+`
		`+ # Rules may optionally specify a scenario to limit applicability.`
		`+ if self._scenario:`
		`+ matching_results = [r for r in matching_results if self.scenario in r['data'].get('scenario', [])]`
		`+`
		`if not matching_results:`
		`return TestResultMissing(item, self.test_case_name)`
		`# If we find multiple matching results, we always use the first one which`
		`@@ -160,13 +165,18 @@`
		`return RuleSatisfied()`
		`return TestResultFailed(item, self.test_case_name, matching_result['id'])`

		`+ @property`
		`+ def _scenario(self):`
		`+ return getattr(self, 'scenario', None)`
		`+`
		`def __repr__(self):`
		`- return "%s(test_case_name=%r)" % (self.__class__.__name__, self.test_case_name)`
		`+ return "%s(test_case_name=%r, scenario=%r)" % (self.__class__.__name__, self.test_case_name, self._scenario)`

		`def to_json(self):`
		`return {`
		`'rule': self.__class__.__name__,`
		`'test_case_name': self.test_case_name,`
		`+ 'scenario': self._scenario,`
		`}`

greenwave/tests/test_policies.py

file modified

+14

		`@@ -190,6 +190,20 @@`
		`assert 'Policies are not configured properly' in str(excinfo.value)`


		`+ def test_passing_testcasename_with_scenario(tmpdir):`
		`+ p = tmpdir.join('fedora.yaml')`
		`+ p.write("""`
		`+ --- !Policy`
		`+ id: "rawhide_compose_sync_to_mirrors"`
		`+ product_versions:`
		`+ - fedora-rawhide`
		`+ decision_context: rawhide_compose_sync_to_mirrors`
		`+ rules:`
		`+ - !PassingTestCaseRule {test_case_name: compose.install_default_upload, scenario: somescenario}`
		`+ """)`
		`+ load_policies(tmpdir.strpath)`
		`+`
		`+`
		`def test_version_endpoint():`
		`app = create_app('greenwave.config.TestingConfig')`
		`test_app = app.test_client()`

ralph commented 6 years ago

I spent some time this afternoon with @mohanboddu and @adamwill
investigating use of greenwave for gating rawhide composes.

https://infrastructure.fedoraproject.org/cgit/ansible.git/commit/?id=f693b6de0d6566881adc189404176edb0bef6121

One thing we ran into was that the requirements for a rawhide compose
can't be expressed only in terms of testcase names. They must be
specified in terms of both the testcase name and the scenario.

This patch gives greenwave the ability to do that. A passing
testcaserule now optionally allows also specifying a required scenario
for that testcase.

I don't yet have a full list of testcase X scenario combinations, but
when I have one, I'll commit it to ansible.

ralph commented 6 years ago