PR#1288: Retry watching koji tasks on server outage - pungi

Haibo Lin • 4 years ago

114df77

pungi/wrappers/kojiwrapper.py

file modified

+7 -3

		`@@ -308,6 +308,10 @@`
		`"""Checks if output indicates connection error."""`
		`return re.search('error: failed to connect\n$', output)`

		`+ def _has_offline_error(self, output):`
		`+ """Check if output indicates server offline."""`
		`+ return re.search('koji: ServerOffline:', output)`
		`+`
		`def _wait_for_task(self, task_id, logfile=None, max_retries=None):`
		`"""Tries to wait for a task to finish. On connection error it will`
		retry with `watch-task` command.
		`@@ -318,8 +322,8 @@`
		`while True:`
		`retcode, output = run(cmd, can_fail=True, logfile=logfile, universal_newlines=True)`

		`- if retcode == 0 or not self._has_connection_error(output):`
		`- # Task finished for reason other than connection error.`
		`+ if retcode == 0 or not (self._has_connection_error(output) or self._has_offline_error(output)):`
		`+ # Task finished for reason other than connection error or server offline error.`
		`return retcode, output`

		`attempt += 1`
		`@@ -345,7 +349,7 @@`
		`% (" ".join(command), output))`
		`task_id = int(match.groups()[0])`

		`- if retcode != 0 and self._has_connection_error(output):`
		`+ if retcode != 0 and (self._has_connection_error(output) or self._has_offline_error(output)):`
		`retcode, output = self._wait_for_task(task_id, logfile=log_file, max_retries=max_retries)`

		`return {`

tests/test_koji_wrapper.py

file modified

+16

		`@@ -701,6 +701,22 @@`
		`can_fail=True, logfile=None, universal_newlines=True)])`
		`self.assertEqual(sleep.mock_calls, [mock.call(i * 10) for i in range(1, 2)])`

		`+ @mock.patch('pungi.wrappers.kojiwrapper.run')`
		`+ def test_server_offline_and_retry(self, run):`
		`+ output = 'Created task: 1234\nkoji: ServerOffline:'`
		`+ retry = 'Created task: 1234\nOook\n'`
		`+ run.side_effect = [(1, output), (0, retry)]`
		`+`
		`+ result = self.koji.run_blocking_cmd('cmd')`
		`+`
		`+ self.assertDictEqual(result, {'retcode': 0, 'output': retry, 'task_id': 1234})`
		`+ self.assertEqual(run.mock_calls,`
		`+ [mock.call('cmd', can_fail=True, logfile=None, env=None,`
		`+ universal_newlines=True),`
		`+ mock.call(['koji', '--profile=custom-koji', 'watch-task', '1234'],`
		`+ can_fail=True, logfile=None, universal_newlines=True)])`
		`+`
		`+`

		`RPM_QA_QF_OUTPUT = """`
		`cjkuni-uming-fonts-0.2.20080216.1-56.fc23.noarch`