From ead80c8c4c815168035b5056b6809df4d8d3fd47 Mon Sep 17 00:00:00 2001 From: Filip Valder Date: Feb 08 2019 19:06:23 +0000 Subject: Rejuvente monitor module --- diff --git a/freshmaker/__init__.py b/freshmaker/__init__.py index 2cbb935..bde7790 100644 --- a/freshmaker/__init__.py +++ b/freshmaker/__init__.py @@ -57,3 +57,6 @@ from freshmaker.auth import init_auth # noqa init_auth(login_manager, conf.auth_backend) from freshmaker import views # noqa + +from freshmaker.monitor import db_hook_event_listeners # noqa +db_hook_event_listeners(target=db.engine) diff --git a/freshmaker/consumer.py b/freshmaker/consumer.py index 26f2fb5..7fbb2c2 100644 --- a/freshmaker/consumer.py +++ b/freshmaker/consumer.py @@ -30,8 +30,8 @@ import moksha.hub from freshmaker import log, conf, messaging, events, app from freshmaker.monitor import ( - messaging_received_counter, messaging_received_ignored_counter, - messaging_received_passed_counter, messaging_received_failed_counter) + messaging_rx_counter, messaging_rx_ignored_counter, + messaging_rx_processed_ok_counter, messaging_rx_failed_counter) from freshmaker.utils import load_classes @@ -85,7 +85,7 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): super(FreshmakerConsumer, self).validate(message) def consume(self, message): - messaging_received_counter.inc() + messaging_rx_counter.inc() # Sometimes, the messages put into our queue are artificially put there # by other parts of our own codebase. If they are already abstracted @@ -100,7 +100,7 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): if not msg: # We do not log here anything, because it would create lot of # useless messages in the logs. - messaging_received_ignored_counter.inc() + messaging_rx_ignored_counter.inc() return # Primary work is done here. @@ -115,9 +115,9 @@ class FreshmakerConsumer(fedmsg.consumers.FedmsgConsumer): # to have global app_context. with app.app_context(): self.process_event(msg) - messaging_received_passed_counter.inc() + messaging_rx_processed_ok_counter.inc() except Exception: - messaging_received_failed_counter.inc() + messaging_rx_failed_counter.inc() log.exception('Failed while handling {0!r}'.format(msg)) if self.stop_condition and self.stop_condition(message): diff --git a/freshmaker/messaging.py b/freshmaker/messaging.py index 83c8b9e..5fe8ad8 100644 --- a/freshmaker/messaging.py +++ b/freshmaker/messaging.py @@ -40,11 +40,25 @@ def publish(topic, msg): :param dict msg: the message contents of the message (typically JSON) :return: the value returned from underlying backend "send" method. """ + from freshmaker.monitor import ( + messaging_tx_to_send_counter, messaging_tx_sent_ok_counter, + messaging_tx_failed_counter) + + messaging_tx_to_send_counter.inc() + try: handler = _messaging_backends[conf.messaging_sender]['publish'] except KeyError: + messaging_tx_failed_counter.inc() raise KeyError("No messaging backend found for %r" % conf.messaging) - return handler(topic, msg) + + try: + rv = handler(topic, msg) + messaging_tx_sent_ok_counter.inc() + return rv + except Exception: + messaging_tx_failed_counter.inc() + raise def _fedmsg_publish(topic, msg): diff --git a/freshmaker/monitor.py b/freshmaker/monitor.py index c60938a..3d93c6c 100644 --- a/freshmaker/monitor.py +++ b/freshmaker/monitor.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2018 Red Hat, Inc. +# Copyright (c) 2019 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -19,42 +19,62 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# For an up-to-date version of this module, see: +# https://pagure.io/monitor-flask-sqlalchemy + import os import tempfile -from freshmaker import db -from flask import Response -from flask.views import MethodView -from prometheus_client import ( +from flask import Blueprint, Response +from prometheus_client import ( # noqa: F401 ProcessCollector, CollectorRegistry, Counter, multiprocess, - Histogram, generate_latest) + Histogram, generate_latest, start_http_server, CONTENT_TYPE_LATEST) from sqlalchemy import event +# Service-specific imports + if not os.environ.get('prometheus_multiproc_dir'): os.environ.setdefault('prometheus_multiproc_dir', tempfile.mkdtemp()) registry = CollectorRegistry() ProcessCollector(registry=registry) multiprocess.MultiProcessCollector(registry) +if os.getenv('MONITOR_STANDALONE_METRICS_SERVER_ENABLE', 'false') == 'true': + port = os.getenv('MONITOR_STANDALONE_METRICS_SERVER_PORT', '10040') + start_http_server(int(port), registry=registry) + # Generic metrics -messaging_received_counter = Counter( - 'messaging_received', +messaging_rx_counter = Counter( + 'messaging_rx', 'Total number of messages received', registry=registry) -messaging_received_ignored_counter = Counter( - 'messaging_received_ignored', +messaging_rx_ignored_counter = Counter( + 'messaging_rx_ignored', 'Number of received messages, which were ignored', registry=registry) -messaging_received_passed_counter = Counter( - 'messaging_received_passed', +messaging_rx_processed_ok_counter = Counter( + 'messaging_rx_processed_ok', 'Number of received messages, which were processed successfully', registry=registry) -messaging_received_failed_counter = Counter( - 'messaging_received_failed', +messaging_rx_failed_counter = Counter( + 'messaging_rx_failed', 'Number of received messages, which failed during processing', registry=registry) +messaging_tx_to_send_counter = Counter( + 'messaging_tx_to_send', + 'Total number of messages to send', + registry=registry) +messaging_tx_sent_ok_counter = Counter( + 'messaging_tx_sent_ok', + 'Number of messages, which were sent successfully', + registry=registry) +messaging_tx_failed_counter = Counter( + 'messaging_tx_failed', + 'Number of messages, for which the sender failed', + registry=registry) + db_dbapi_error_counter = Counter( 'db_dbapi_error', 'Number of DBAPI errors', @@ -67,20 +87,12 @@ db_handle_error_counter = Counter( 'db_handle_error', 'Number of exceptions during connection', registry=registry) -db_transaction_begin_counter = Counter( - 'db_transaction_begin', - 'Number of started transactions', - registry=registry) -db_transaction_commit_counter = Counter( - 'db_transaction_commit', - 'Number of transactions, which were committed', - registry=registry) db_transaction_rollback_counter = Counter( 'db_transaction_rollback', 'Number of transactions, which were rolled back', registry=registry) -# Freshmaker-specific metrics +# Service-specific metrics freshmaker_artifact_build_done_counter = Counter( 'freshmaker_artifact_build_done', 'Number of successful artifact builds', @@ -115,45 +127,36 @@ freshmaker_event_api_latency = Histogram( 'EventAPI latency', registry=registry) -@event.listens_for(db.engine, 'dbapi_error', named=True) -def receive_dbapi_error(**kw): - db_dbapi_error_counter.inc() - - -@event.listens_for(db.engine, 'engine_connect') -def receive_engine_connect(conn, branch): - db_engine_connect_counter.inc() - - -@event.listens_for(db.engine, 'handle_error') -def receive_handle_error(exception_context): - db_handle_error_counter.inc() +def db_hook_event_listeners(target=None): + # Service-specific import of db + from freshmaker import db + if not target: + target = db.engine -@event.listens_for(db.engine, 'begin') -def receive_begin(conn): - db_transaction_begin_counter.inc() + @event.listens_for(target, 'dbapi_error', named=True) + def receive_dbapi_error(**kw): + db_dbapi_error_counter.inc() + @event.listens_for(target, 'engine_connect') + def receive_engine_connect(conn, branch): + db_engine_connect_counter.inc() -@event.listens_for(db.engine, 'commit') -def receive_commit(conn): - db_transaction_commit_counter.inc() + @event.listens_for(target, 'handle_error') + def receive_handle_error(exception_context): + db_handle_error_counter.inc() + @event.listens_for(target, 'rollback') + def receive_rollback(conn): + db_transaction_rollback_counter.inc() -@event.listens_for(db.engine, 'rollback') -def receive_rollback(conn): - db_transaction_rollback_counter.inc() +monitor_api = Blueprint( + 'monitor', __name__, + url_prefix='/api/1/monitor') -class MonitorAPI(MethodView): - rest_api_v1 = { - 'basic': { - 'url': '/api/1/monitor/metrics/', - 'options': { - 'methods': ['GET'], - } - } - } - def get(self): - return Response(generate_latest(registry)) +@monitor_api.route('/metrics') +def metrics(): + return Response(generate_latest(registry), + content_type=CONTENT_TYPE_LATEST) diff --git a/freshmaker/views.py b/freshmaker/views.py index e914117..bfdefdb 100644 --- a/freshmaker/views.py +++ b/freshmaker/views.py @@ -40,7 +40,7 @@ from freshmaker.api_utils import pagination_metadata from freshmaker.auth import login_required, requires_role, require_scopes from freshmaker.parsers.internal.manual_rebuild import FreshmakerManualRebuildParser from freshmaker.monitor import ( - MonitorAPI, freshmaker_build_api_latency, freshmaker_event_api_latency) + monitor_api, freshmaker_build_api_latency, freshmaker_event_api_latency) api_v1 = { 'event_types': { @@ -124,7 +124,6 @@ api_v1 = { } }, }, - 'monitor': MonitorAPI.rest_api_v1, 'about': { 'about': { 'url': '/api/1/about/', @@ -301,7 +300,6 @@ API_V1_MAPPING = { 'event_types': EventTypeAPI, 'build_types': BuildTypeAPI, 'build_states': BuildStateAPI, - 'monitor': MonitorAPI, 'about': AboutAPI, } @@ -316,5 +314,7 @@ def register_api_v1(): view_func=view, **val['options']) + app.register_blueprint(monitor_api) + register_api_v1() diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 21c0e39..04a1436 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (c) 2017 Red Hat, Inc. +# Copyright (c) 2019 Red Hat, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -19,12 +19,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import os import mock +import pytest import freshmaker +import requests +from six.moves import reload_module from freshmaker import app, db, events, models, login_manager from tests import helpers +num_of_metrics = 44 + @login_manager.user_loader def user_loader(username): @@ -48,13 +54,10 @@ class TestViews(helpers.ModelsTestCase): db.session.expire_all() def test_monitor_api_structure(self): - resp = self.client.get('/api/1/monitor/metrics/') - self.assertEqual( - len([l.startswith('# TYPE') - for l in resp.get_data(as_text=True).splitlines()]), 140) + resp = self.client.get('/api/1/monitor/metrics') self.assertEqual( - len([l.startswith('# HELP') - for l in resp.get_data(as_text=True).splitlines()]), 140) + len([l for l in resp.get_data(as_text=True).splitlines() + if l.startswith('# TYPE')]), num_of_metrics) class ConsumerTest(helpers.ModelsTestCase): @@ -80,7 +83,7 @@ class ConsumerTest(helpers.ModelsTestCase): return msg def _get_monitor_value(self, key): - resp = self.client.get('/api/1/monitor/metrics/') + resp = self.client.get('/api/1/monitor/metrics') for line in resp.get_data(as_text=True).splitlines(): k, v = line.split(" ")[:2] if k == key: @@ -99,7 +102,7 @@ class ConsumerTest(helpers.ModelsTestCase): global_consumer.return_value = consumer handle.return_value = [freshmaker.events.TestingEvent("ModuleBuilt handled")] - prev_counter_value = self._get_monitor_value("messaging_received_passed_total") + prev_counter_value = self._get_monitor_value("messaging_rx_processed_ok_total") msg = self._module_state_change_msg() consumer.consume(msg) @@ -107,5 +110,20 @@ class ConsumerTest(helpers.ModelsTestCase): event = consumer.incoming.get() self.assertEqual(event.msg_id, "ModuleBuilt handled") - counter_value = self._get_monitor_value("messaging_received_passed_total") + counter_value = self._get_monitor_value("messaging_rx_processed_ok_total") self.assertEqual(prev_counter_value + 1, counter_value) + + +def test_standalone_metrics_server_disabled_by_default(): + with pytest.raises(requests.exceptions.ConnectionError): + requests.get('http://127.0.0.1:10040/metrics') + + +def test_standalone_metrics_server(): + os.environ['MONITOR_STANDALONE_METRICS_SERVER_ENABLE'] = 'true' + reload_module(freshmaker.monitor) + + r = requests.get('http://127.0.0.1:10040/metrics') + + assert len([l for l in r.text.splitlines() + if l.startswith('# TYPE')]) == num_of_metrics