From e23e2dd22caef1bd1f5f6073e0cf86a50126d95d Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Apr 22 2020 10:44:29 +0000 Subject: ODCS client: Add --watch argument to watch the compose log. --- diff --git a/client/contrib/odcs b/client/contrib/odcs index d454516..e6459b8 100755 --- a/client/contrib/odcs +++ b/client/contrib/odcs @@ -66,6 +66,9 @@ parser.add_argument( parser.add_argument( '-q', '--quiet', action='store_true', help='Run without detailed log messages') +parser.add_argument( + '--watch', action='store_true', + help="Watch compose logs") subparsers = parser.add_subparsers( description='These commands you can use to operate composes with ODCS') @@ -214,6 +217,9 @@ wait_parser.set_defaults(command='wait') wait_parser.add_argument( 'compose_id', default=None, help="ODCS compose id") +wait_parser.add_argument( + '--watch', action='store_true', + help="Watch compose logs") delete_parser = subparsers.add_parser( @@ -356,5 +362,9 @@ else: if not args.quiet: print("Waiting for command %s on compose %d to finish." % (args.command, result["id"])) - result = client.wait_for_compose(result["id"], 3600) + try: + result = client.wait_for_compose(result["id"], 3600, watch_logs=args.watch) + except (KeyboardInterrupt, SystemExit): + pass + print(json.dumps(result, indent=4, sort_keys=True)) diff --git a/client/odcs/client/odcs.py b/client/odcs/client/odcs.py index 0015f85..1f3d4f2 100644 --- a/client/odcs/client/odcs.py +++ b/client/odcs/client/odcs.py @@ -21,6 +21,7 @@ # # Written by Chenxiong Qi +import os import json import requests import time @@ -40,6 +41,42 @@ class AuthMech(object): return mech in (cls.OpenIDC, cls.Kerberos, cls.Anonymous, cls.SSL) +class ComposeLog(object): + def __init__(self, compose): + """ + Creates new ComposeLog instance. + """ + self.url = os.path.join(compose["toplevel_url"], "pungi-stderr.log") + self.offset = 0 + + def read(self): + """ + Reads the Compose log from the ODCS server and returns its content. + + This method can be called repeatedly to get the latest content in + the log. Similar to "tail -f log". + + :return str: New log lines or None if log does not exist (yet) on the + ODCS server. + """ + headers = {"Range": "bytes=%d-" % self.offset} + r = requests.get(self.url, headers=headers) + + # Log does not exists yet on the ODCS server. + if r.status_code == 404: + return None + + # 416 Range Not Satisfiable - nothing new in log. + if r.status_code == 416: + return "" + + r.raise_for_status() + + content = r.text + self.offset += len(content) + return content + + def validate_int(value, min=1, type_error=None, value_error=None): if not isinstance(value, int): if type_error: @@ -484,7 +521,7 @@ class ODCS(object): r = self._get('composes/{0}'.format(compose_id)) return r.json() - def wait_for_compose(self, compose_id, timeout=300): + def wait_for_compose(self, compose_id, timeout=300, watch_logs=False): """ Polls the ODCS server repeatedly to find out whether the compose moved from "wait" or "generating" state to some final state. Blocks @@ -498,14 +535,26 @@ class ODCS(object): :param int compose_id: compose ID. :param int timeout: Number of seconds to wait/block. + :param bool watch_logs: If True, this method prints the compose log to + stdout every 10 seconds while waiting for the compose to finish. :rtype: dict :return: a mapping representing a compose """ elapsed = 0 - sleep_time = 1 + if watch_logs: + sleep_time = 10 + compose = self.get_compose(compose_id) + log = ComposeLog(compose) + else: + sleep_time = 1 + log = None start_time = time.time() while True: compose = self.get_compose(compose_id) + if log: + data = log.read() + if data: + print(data) if compose['state_name'] not in ['wait', 'generating']: return compose @@ -520,6 +569,10 @@ class ODCS(object): # Increase the sleep time for next try. But do not try sleeping # longer than the `timeout`. elapsed = time.time() - start_time - sleep_time = round(sleep_time * 1.5) + + # Do not increase sleep time in case we are watching logs. + if not watch_logs: + sleep_time = round(sleep_time * 1.5) + if elapsed + sleep_time > timeout: sleep_time = timeout - elapsed diff --git a/client/tests/test_client_odcs.py b/client/tests/test_client_odcs.py index 6ee1c1a..c60f4ec 100644 --- a/client/tests/test_client_odcs.py +++ b/client/tests/test_client_odcs.py @@ -23,14 +23,15 @@ import json import unittest +import copy import mock -from mock import patch, Mock +from mock import patch, Mock, MagicMock from odcs.client.odcs import AuthMech from odcs.client.odcs import ( ODCS, ComposeSourceTag, ComposeSourceModule, ComposeSourcePulp, - ComposeSourceRawConfig, ComposeSourceBuild) + ComposeSourceRawConfig, ComposeSourceBuild, ComposeLog) from odcs.client.odcs import validate_int @@ -529,3 +530,67 @@ class TestWaitForCompose(unittest.TestCase): self.assertEqual(sleep.mock_calls, [mock.call(1), mock.call(2), mock.call(3), mock.call(4)]) + + @patch("odcs.client.odcs.ComposeLog.read") + def test_wait_for_compose_watch_logs(self, log_read, get_compose, sleep): + get_compose.side_effect = [ + { + "state_name": "wait", + "toplevel_url": "http://localhost/composes/odcs-1" + }, + { + "state_name": "generating", + "toplevel_url": "http://localhost/composes/odcs-1" + }, + { + "state_name": "done", + "toplevel_url": "http://localhost/composes/odcs-1" + }, + ] + log_read.side_effect = [None, "line\n"] + self.odcs.wait_for_compose(1, watch_logs=True) + + self.assertEqual(sleep.mock_calls, + [mock.call(10)]) + self.assertEqual(get_compose.mock_calls, + [mock.call(1)] * 3) + self.assertEqual(len(log_read.mock_calls), 2) + + +@patch('odcs.client.odcs.requests') +class TestComposeLog(unittest.TestCase): + """Test ODCS.wait_for_compose""" + + def setUp(self): + compose = { + 'toplevel_url': 'http://localhost/composes/odcs-1' + } + self.compose_log = ComposeLog(compose) + + def test_compose_log_404(self, requests): + requests.get.return_value.status_code = 404 + ret = self.compose_log.read() + requests.get.assert_called_once_with( + 'http://localhost/composes/odcs-1/pungi-stderr.log', + headers={'Range': 'bytes=0-'} + ) + self.assertEqual(ret, None) + + def test_compose_log_multiple_calls(self, requests): + responses = [ + MagicMock(status_code=200, text="line\n"), + MagicMock(status_code=200, text="another line\n"), + MagicMock(status_code=416, text=""), + MagicMock(status_code=200, text="another line\n") + ] + requests.get.side_effect = responses + length = 0 + for m in copy.copy(responses): + ret = self.compose_log.read() + self.assertEqual(ret, m.text) + requests.get.assert_called_once_with( + 'http://localhost/composes/odcs-1/pungi-stderr.log', + headers={'Range': 'bytes=%d-' % length} + ) + requests.get.reset_mock() + length += len(ret)