#365 Rejuvente monitor module
Merged 5 months ago by jkaluza. Opened 5 months ago by fivaldi.

file modified
+3

@@ -57,3 +57,6 @@ 

  init_auth(login_manager, conf.auth_backend)

  

  from freshmaker import views  # noqa

+ 

+ from freshmaker.monitor import db_hook_event_listeners  # noqa

+ db_hook_event_listeners(target=db.engine)

file modified
+6 -6

@@ -30,8 +30,8 @@ 

  

  from freshmaker import log, conf, messaging, events, app

  from freshmaker.monitor import (

-     messaging_received_counter, messaging_received_ignored_counter,

-     messaging_received_passed_counter, messaging_received_failed_counter)

+     messaging_rx_counter, messaging_rx_ignored_counter,

+     messaging_rx_processed_ok_counter, messaging_rx_failed_counter)

  from freshmaker.utils import load_classes

  

  

@@ -85,7 +85,7 @@ 

              super(FreshmakerConsumer, self).validate(message)

  

      def consume(self, message):

-         messaging_received_counter.inc()

+         messaging_rx_counter.inc()

  

          # Sometimes, the messages put into our queue are artificially put there

          # by other parts of our own codebase.  If they are already abstracted

@@ -100,7 +100,7 @@ 

          if not msg:

              # We do not log here anything, because it would create lot of

              # useless messages in the logs.

-             messaging_received_ignored_counter.inc()

+             messaging_rx_ignored_counter.inc()

              return

  

          # Primary work is done here.

@@ -115,9 +115,9 @@ 

              # to have global app_context.

              with app.app_context():

                  self.process_event(msg)

-             messaging_received_passed_counter.inc()

+             messaging_rx_processed_ok_counter.inc()

          except Exception:

-             messaging_received_failed_counter.inc()

+             messaging_rx_failed_counter.inc()

              log.exception('Failed while handling {0!r}'.format(msg))

  

          if self.stop_condition and self.stop_condition(message):

file modified
+15 -1

@@ -40,11 +40,25 @@ 

      :param dict msg: the message contents of the message (typically JSON)

      :return: the value returned from underlying backend "send" method.

      """

+     from freshmaker.monitor import (

+         messaging_tx_to_send_counter, messaging_tx_sent_ok_counter,

+         messaging_tx_failed_counter)

+ 

+     messaging_tx_to_send_counter.inc()

+ 

      try:

          handler = _messaging_backends[conf.messaging_sender]['publish']

      except KeyError:

+         messaging_tx_failed_counter.inc()

          raise KeyError("No messaging backend found for %r" % conf.messaging)

-     return handler(topic, msg)

+ 

+     try:

+         rv = handler(topic, msg)

+         messaging_tx_sent_ok_counter.inc()

+         return rv

+     except Exception:

+         messaging_tx_failed_counter.inc()

+         raise

  

  

  def _fedmsg_publish(topic, msg):

file modified
+59 -56

@@ -1,5 +1,5 @@ 

  # -*- coding: utf-8 -*-

- # Copyright (c) 2018  Red Hat, Inc.

+ # Copyright (c) 2019  Red Hat, Inc.

  #

  # Permission is hereby granted, free of charge, to any person obtaining a copy

  # of this software and associated documentation files (the "Software"), to deal

@@ -19,42 +19,62 @@ 

  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  # SOFTWARE.

  

+ # For an up-to-date version of this module, see:

+ #   https://pagure.io/monitor-flask-sqlalchemy

+ 

  import os

  import tempfile

  

- from freshmaker import db

- from flask import Response

- from flask.views import MethodView

- from prometheus_client import (

+ from flask import Blueprint, Response

+ from prometheus_client import (  # noqa: F401

      ProcessCollector, CollectorRegistry, Counter, multiprocess,

-     Histogram, generate_latest)

+     Histogram, generate_latest, start_http_server, CONTENT_TYPE_LATEST)

  from sqlalchemy import event

  

+ # Service-specific imports

+ 

  

  if not os.environ.get('prometheus_multiproc_dir'):

      os.environ.setdefault('prometheus_multiproc_dir', tempfile.mkdtemp())

  registry = CollectorRegistry()

  ProcessCollector(registry=registry)

  multiprocess.MultiProcessCollector(registry)

+ if os.getenv('MONITOR_STANDALONE_METRICS_SERVER_ENABLE', 'false') == 'true':

+     port = os.getenv('MONITOR_STANDALONE_METRICS_SERVER_PORT', '10040')

+     start_http_server(int(port), registry=registry)

+ 

  

  # Generic metrics

- messaging_received_counter = Counter(

-     'messaging_received',

+ messaging_rx_counter = Counter(

+     'messaging_rx',

      'Total number of messages received',

      registry=registry)

- messaging_received_ignored_counter = Counter(

-     'messaging_received_ignored',

+ messaging_rx_ignored_counter = Counter(

+     'messaging_rx_ignored',

      'Number of received messages, which were ignored',

      registry=registry)

- messaging_received_passed_counter = Counter(

-     'messaging_received_passed',

+ messaging_rx_processed_ok_counter = Counter(

+     'messaging_rx_processed_ok',

      'Number of received messages, which were processed successfully',

      registry=registry)

- messaging_received_failed_counter = Counter(

-     'messaging_received_failed',

+ messaging_rx_failed_counter = Counter(

+     'messaging_rx_failed',

      'Number of received messages, which failed during processing',

      registry=registry)

  

+ messaging_tx_to_send_counter = Counter(

+     'messaging_tx_to_send',

+     'Total number of messages to send',

+     registry=registry)

+ messaging_tx_sent_ok_counter = Counter(

+     'messaging_tx_sent_ok',

+     'Number of messages, which were sent successfully',

+     registry=registry)

+ messaging_tx_failed_counter = Counter(

+     'messaging_tx_failed',

+     'Number of messages, for which the sender failed',

+     registry=registry)

+ 

  db_dbapi_error_counter = Counter(

      'db_dbapi_error',

      'Number of DBAPI errors',

@@ -67,20 +87,12 @@ 

      'db_handle_error',

      'Number of exceptions during connection',

      registry=registry)

- db_transaction_begin_counter = Counter(

-     'db_transaction_begin',

-     'Number of started transactions',

-     registry=registry)

- db_transaction_commit_counter = Counter(

-     'db_transaction_commit',

-     'Number of transactions, which were committed',

-     registry=registry)

  db_transaction_rollback_counter = Counter(

      'db_transaction_rollback',

      'Number of transactions, which were rolled back',

      registry=registry)

  

- # Freshmaker-specific metrics

+ # Service-specific metrics

  freshmaker_artifact_build_done_counter = Counter(

      'freshmaker_artifact_build_done',

      'Number of successful artifact builds',

@@ -115,45 +127,36 @@ 

      'EventAPI latency', registry=registry)

  

  

- @event.listens_for(db.engine, 'dbapi_error', named=True)

- def receive_dbapi_error(**kw):

-     db_dbapi_error_counter.inc()

- 

- 

- @event.listens_for(db.engine, 'engine_connect')

- def receive_engine_connect(conn, branch):

-     db_engine_connect_counter.inc()

- 

- 

- @event.listens_for(db.engine, 'handle_error')

- def receive_handle_error(exception_context):

-     db_handle_error_counter.inc()

+ def db_hook_event_listeners(target=None):

+     # Service-specific import of db

+     from freshmaker import db

  

+     if not target:

+         target = db.engine

  

- @event.listens_for(db.engine, 'begin')

- def receive_begin(conn):

-     db_transaction_begin_counter.inc()

+     @event.listens_for(target, 'dbapi_error', named=True)

+     def receive_dbapi_error(**kw):

+         db_dbapi_error_counter.inc()

  

+     @event.listens_for(target, 'engine_connect')

+     def receive_engine_connect(conn, branch):

+         db_engine_connect_counter.inc()

  

- @event.listens_for(db.engine, 'commit')

- def receive_commit(conn):

-     db_transaction_commit_counter.inc()

+     @event.listens_for(target, 'handle_error')

+     def receive_handle_error(exception_context):

+         db_handle_error_counter.inc()

  

+     @event.listens_for(target, 'rollback')

+     def receive_rollback(conn):

+         db_transaction_rollback_counter.inc()

  

- @event.listens_for(db.engine, 'rollback')

- def receive_rollback(conn):

-     db_transaction_rollback_counter.inc()

  

+ monitor_api = Blueprint(

+     'monitor', __name__,

+     url_prefix='/api/1/monitor')

  

- class MonitorAPI(MethodView):

-     rest_api_v1 = {

-         'basic': {

-             'url': '/api/1/monitor/metrics/',

-             'options': {

-                 'methods': ['GET'],

-             }

-         }

-     }

  

-     def get(self):

-         return Response(generate_latest(registry))

+ @monitor_api.route('/metrics')

+ def metrics():

+     return Response(generate_latest(registry),

+                     content_type=CONTENT_TYPE_LATEST)

file modified
+3 -3

@@ -40,7 +40,7 @@ 

  from freshmaker.auth import login_required, requires_role, require_scopes

  from freshmaker.parsers.internal.manual_rebuild import FreshmakerManualRebuildParser

  from freshmaker.monitor import (

-     MonitorAPI, freshmaker_build_api_latency, freshmaker_event_api_latency)

+     monitor_api, freshmaker_build_api_latency, freshmaker_event_api_latency)

  

  api_v1 = {

      'event_types': {

@@ -124,7 +124,6 @@ 

              }

          },

      },

-     'monitor': MonitorAPI.rest_api_v1,

      'about': {

          'about': {

              'url': '/api/1/about/',

@@ -301,7 +300,6 @@ 

      'event_types': EventTypeAPI,

      'build_types': BuildTypeAPI,

      'build_states': BuildStateAPI,

-     'monitor': MonitorAPI,

      'about': AboutAPI,

  }

  

@@ -316,5 +314,7 @@ 

                               view_func=view,

                               **val['options'])

  

+     app.register_blueprint(monitor_api)

+ 

  

  register_api_v1()

file modified
+28 -10

@@ -1,5 +1,5 @@ 

  # -*- coding: utf-8 -*-

- # Copyright (c) 2017  Red Hat, Inc.

+ # Copyright (c) 2019  Red Hat, Inc.

  #

  # Permission is hereby granted, free of charge, to any person obtaining a copy

  # of this software and associated documentation files (the "Software"), to deal

@@ -19,12 +19,18 @@ 

  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

  # SOFTWARE.

  

+ import os

  import mock

+ import pytest

  import freshmaker

+ import requests

  

+ from six.moves import reload_module

  from freshmaker import app, db, events, models, login_manager

  from tests import helpers

  

+ num_of_metrics = 44

+ 

  

  @login_manager.user_loader

  def user_loader(username):

@@ -48,13 +54,10 @@ 

          db.session.expire_all()

  

      def test_monitor_api_structure(self):

-         resp = self.client.get('/api/1/monitor/metrics/')

-         self.assertEqual(

-             len([l.startswith('# TYPE')

-                  for l in resp.get_data(as_text=True).splitlines()]), 140)

+         resp = self.client.get('/api/1/monitor/metrics')

          self.assertEqual(

-             len([l.startswith('# HELP')

-                  for l in resp.get_data(as_text=True).splitlines()]), 140)

+             len([l for l in resp.get_data(as_text=True).splitlines()

+                  if l.startswith('# TYPE')]), num_of_metrics)

  

  

  class ConsumerTest(helpers.ModelsTestCase):

@@ -80,7 +83,7 @@ 

          return msg

  

      def _get_monitor_value(self, key):

-         resp = self.client.get('/api/1/monitor/metrics/')

+         resp = self.client.get('/api/1/monitor/metrics')

          for line in resp.get_data(as_text=True).splitlines():

              k, v = line.split(" ")[:2]

              if k == key:

@@ -99,7 +102,7 @@ 

          global_consumer.return_value = consumer

          handle.return_value = [freshmaker.events.TestingEvent("ModuleBuilt handled")]

  

-         prev_counter_value = self._get_monitor_value("messaging_received_passed_total")

+         prev_counter_value = self._get_monitor_value("messaging_rx_processed_ok_total")

  

          msg = self._module_state_change_msg()

          consumer.consume(msg)

@@ -107,5 +110,20 @@ 

          event = consumer.incoming.get()

          self.assertEqual(event.msg_id, "ModuleBuilt handled")

  

-         counter_value = self._get_monitor_value("messaging_received_passed_total")

+         counter_value = self._get_monitor_value("messaging_rx_processed_ok_total")

          self.assertEqual(prev_counter_value + 1, counter_value)

+ 

+ 

+ def test_standalone_metrics_server_disabled_by_default():

+     with pytest.raises(requests.exceptions.ConnectionError):

+         requests.get('http://127.0.0.1:10040/metrics')

+ 

+ 

+ def test_standalone_metrics_server():

+     os.environ['MONITOR_STANDALONE_METRICS_SERVER_ENABLE'] = 'true'

+     reload_module(freshmaker.monitor)

+ 

+     r = requests.get('http://127.0.0.1:10040/metrics')

+ 

+     assert len([l for l in r.text.splitlines()

+                 if l.startswith('# TYPE')]) == num_of_metrics

  • Fix metrics names to be in sync with other F2.0 projects
  • Add TX messaging counters
  • Rejuvenate monitor module
  • Make corrections to tests, add tests for standalone server

pretty please pagure-ci rebuild

5 months ago

Commit 92534be fixes this pull-request

Pull-Request has been merged by jkaluza

5 months ago

Pull-Request has been merged by jkaluza

5 months ago