PR#373: ODCS client: Add --watch argument to watch the compose log. - odcs

odcs

#373 ODCS client: Add --watch argument to watch the compose log.

Merged 3 years ago by lsedlar. Opened 3 years ago by jkaluza.

jkaluza/odcs watch-logs into master

ODCS client: Add --watch argument to watch the compose log.

Jan Kaluza • 3 years ago

e23e2dd

client/contrib/odcs

file modified

+11 -1

		`@@ -66,6 +66,9 @@`
		`parser.add_argument(`
		`'-q', '--quiet', action='store_true',`
		`help='Run without detailed log messages')`
		`+ parser.add_argument(`
		`+ '--watch', action='store_true',`
		`+ help="Watch compose logs")`

		`subparsers = parser.add_subparsers(`
		`description='These commands you can use to operate composes with ODCS')`
		`@@ -214,6 +217,9 @@`
		`wait_parser.add_argument(`
		`'compose_id', default=None,`
		`help="ODCS compose id")`
		`+ wait_parser.add_argument(`
		`+ '--watch', action='store_true',`
		`+ help="Watch compose logs")`


		`delete_parser = subparsers.add_parser(`
		`@@ -356,5 +362,9 @@`
		`if not args.quiet:`
		`print("Waiting for command %s on compose %d to finish." %`
		`(args.command, result["id"]))`
		`- result = client.wait_for_compose(result["id"], 3600)`
		`+ try:`
		`+ result = client.wait_for_compose(result["id"], 3600, watch_logs=args.watch)`
		`+ except (KeyboardInterrupt, SystemExit):`
		`+ pass`
		`+`
		`print(json.dumps(result, indent=4, sort_keys=True))`

client/odcs/client/odcs.py

file modified

+56 -3

		`@@ -21,6 +21,7 @@`
		`#`
		`# Written by Chenxiong Qi <cqi@redhat.com>`

		`+ import os`
		`import json`
		`import requests`
		`import time`
		`@@ -40,6 +41,42 @@`
		`return mech in (cls.OpenIDC, cls.Kerberos, cls.Anonymous, cls.SSL)`


		`+ class ComposeLog(object):`
		`+ def __init__(self, compose):`
		`+ """`
		`+ Creates new ComposeLog instance.`
		`+ """`
		`+ self.url = os.path.join(compose["toplevel_url"], "pungi-stderr.log")`
		`+ self.offset = 0`
		`+`
		`+ def read(self):`
		`+ """`
		`+ Reads the Compose log from the ODCS server and returns its content.`
		`+`
		`+ This method can be called repeatedly to get the latest content in`
		`+ the log. Similar to "tail -f log".`
		`+`
		`+ :return str: New log lines or None if log does not exist (yet) on the`
		`+ ODCS server.`
		`+ """`
		`+ headers = {"Range": "bytes=%d-" % self.offset}`
		`+ r = requests.get(self.url, headers=headers)`
		`+`
		`+ # Log does not exists yet on the ODCS server.`
		`+ if r.status_code == 404:`
		`+ return None`
		`+`
		`+ # 416 Range Not Satisfiable - nothing new in log.`
		`+ if r.status_code == 416:`
		`+ return ""`
		`+`
		`+ r.raise_for_status()`
		`+`
		`+ content = r.text`
		`+ self.offset += len(content)`
		`+ return content`
		`+`
		`+`
		`def validate_int(value, min=1, type_error=None, value_error=None):`
		`if not isinstance(value, int):`
		`if type_error:`
		`@@ -484,7 +521,7 @@`
		`r = self._get('composes/{0}'.format(compose_id))`
		`return r.json()`

		`- def wait_for_compose(self, compose_id, timeout=300):`
		`+ def wait_for_compose(self, compose_id, timeout=300, watch_logs=False):`
		`"""`
		`Polls the ODCS server repeatedly to find out whether the compose`
		`moved from "wait" or "generating" state to some final state. Blocks`
		`@@ -498,14 +535,26 @@`

		`:param int compose_id: compose ID.`
		`:param int timeout: Number of seconds to wait/block.`
		`+ :param bool watch_logs: If True, this method prints the compose log to`
		`+ stdout every 10 seconds while waiting for the compose to finish.`
		`:rtype: dict`
		`:return: a mapping representing a compose`
		`"""`
		`elapsed = 0`
		`- sleep_time = 1`
		`+ if watch_logs:`
		`+ sleep_time = 10`
		`+ compose = self.get_compose(compose_id)`
		`+ log = ComposeLog(compose)`
		`+ else:`
		`+ sleep_time = 1`
		`+ log = None`
		`start_time = time.time()`
		`while True:`
		`compose = self.get_compose(compose_id)`
		`+ if log:`
		`+ data = log.read()`
		`+ if data:`
		`+ print(data)`
		`if compose['state_name'] not in ['wait', 'generating']:`
		`return compose`

		`@@ -520,6 +569,10 @@`
		`# Increase the sleep time for next try. But do not try sleeping`
		# longer than the `timeout`.
		`elapsed = time.time() - start_time`
		`- sleep_time = round(sleep_time * 1.5)`
		`+`
		`+ # Do not increase sleep time in case we are watching logs.`
		`+ if not watch_logs:`
		`+ sleep_time = round(sleep_time * 1.5)`
		`+`
		`if elapsed + sleep_time > timeout:`
		`sleep_time = timeout - elapsed`

client/tests/test_client_odcs.py

file modified

+67 -2

		`@@ -23,14 +23,15 @@`

		`import json`
		`import unittest`
		`+ import copy`

		`import mock`
		`- from mock import patch, Mock`
		`+ from mock import patch, Mock, MagicMock`

		`from odcs.client.odcs import AuthMech`
		`from odcs.client.odcs import (`
		`ODCS, ComposeSourceTag, ComposeSourceModule, ComposeSourcePulp,`
		`- ComposeSourceRawConfig, ComposeSourceBuild)`
		`+ ComposeSourceRawConfig, ComposeSourceBuild, ComposeLog)`
		`from odcs.client.odcs import validate_int`


		`@@ -529,3 +530,67 @@`

		`self.assertEqual(sleep.mock_calls,`
		`[mock.call(1), mock.call(2), mock.call(3), mock.call(4)])`
		`+`
		`+ @patch("odcs.client.odcs.ComposeLog.read")`
		`+ def test_wait_for_compose_watch_logs(self, log_read, get_compose, sleep):`
		`+ get_compose.side_effect = [`
		`+ {`
		`+ "state_name": "wait",`
		`+ "toplevel_url": "http://localhost/composes/odcs-1"`
		`+ },`
		`+ {`
		`+ "state_name": "generating",`
		`+ "toplevel_url": "http://localhost/composes/odcs-1"`
		`+ },`
		`+ {`
		`+ "state_name": "done",`
		`+ "toplevel_url": "http://localhost/composes/odcs-1"`
		`+ },`
		`+ ]`
		`+ log_read.side_effect = [None, "line\n"]`
		`+ self.odcs.wait_for_compose(1, watch_logs=True)`
		`+`
		`+ self.assertEqual(sleep.mock_calls,`
		`+ [mock.call(10)])`
		`+ self.assertEqual(get_compose.mock_calls,`
		`+ [mock.call(1)] * 3)`
		`+ self.assertEqual(len(log_read.mock_calls), 2)`
		`+`
		`+`
		`+ @patch('odcs.client.odcs.requests')`
		`+ class TestComposeLog(unittest.TestCase):`
		`+ """Test ODCS.wait_for_compose"""`
		`+`
		`+ def setUp(self):`
		`+ compose = {`
		`+ 'toplevel_url': 'http://localhost/composes/odcs-1'`
		`+ }`
		`+ self.compose_log = ComposeLog(compose)`
		`+`
		`+ def test_compose_log_404(self, requests):`
		`+ requests.get.return_value.status_code = 404`
		`+ ret = self.compose_log.read()`
		`+ requests.get.assert_called_once_with(`
		`+ 'http://localhost/composes/odcs-1/pungi-stderr.log',`
		`+ headers={'Range': 'bytes=0-'}`
		`+ )`
		`+ self.assertEqual(ret, None)`
		`+`
		`+ def test_compose_log_multiple_calls(self, requests):`
		`+ responses = [`
		`+ MagicMock(status_code=200, text="line\n"),`
		`+ MagicMock(status_code=200, text="another line\n"),`
		`+ MagicMock(status_code=416, text=""),`
		`+ MagicMock(status_code=200, text="another line\n")`
		`+ ]`
		`+ requests.get.side_effect = responses`
		`+ length = 0`
		`+ for m in copy.copy(responses):`
		`+ ret = self.compose_log.read()`
		`+ self.assertEqual(ret, m.text)`
		`+ requests.get.assert_called_once_with(`
		`+ 'http://localhost/composes/odcs-1/pungi-stderr.log',`
		`+ headers={'Range': 'bytes=%d-' % length}`
		`+ )`
		`+ requests.get.reset_mock()`
		`+ length += len(ret)`

no initial comment

lsedlar commented on line 37 of client/odcs/client/odcs.py 3 years ago

Is this intended to be included here? If so, maybe some additional info might be useful.

rebased onto 4a5cd7c64ffe476161baec408b63225f8fc9e1f4

3 years ago

jkaluza commented 3 years ago

I've removed that ^ print, added tests and handling of special handling for 404 (this happens when the pungi is not running yet and therefore no log is available).

lsedlar commented 3 years ago

Looks good to me. I guess the missing logs case also applies to pulp composes.

rebased onto e23e2dd

3 years ago