#90 Add support for OpenTelemetry
Merged 9 months ago by lsedlar. Opened 10 months ago by lsedlar.
lsedlar/cts opentelemetry  into  master

file modified
+4
@@ -44,6 +44,10 @@ 

          python3-pyyaml \

          python3-sqlalchemy \

          python3-systemd \

+         python3-opentelemetry-sdk \

+         python3-opentelemetry-instrumentation-flask \

+         python3-opentelemetry-instrumentation-sqlalchemy \

+         python3-opentelemetry-exporter-otlp-proto-http \

          systemd \

          # Debugging packages \

          net-tools \

file modified
+26
@@ -21,6 +21,7 @@ 

  #

  # Written by Jan Kaluza <jkaluza@redhat.com>

  

+ import os

  from logging import getLogger

  

  from flask import Flask, jsonify
@@ -28,6 +29,15 @@ 

  from flask_sqlalchemy import SQLAlchemy

  from werkzeug.exceptions import BadRequest, Unauthorized, NotFound as WerkzeugNotFound

  

+ from opentelemetry import trace

+ from opentelemetry.sdk.resources import Resource, SERVICE_NAME

+ from opentelemetry.sdk.trace import TracerProvider

+ from opentelemetry.sdk.trace.export import ConsoleSpanExporter

+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter

+ from opentelemetry.sdk.trace.export import BatchSpanProcessor

+ from opentelemetry.instrumentation.flask import FlaskInstrumentor

+ from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor

+ 

  from cts.logger import init_logging

  from cts.config import init_config

  from cts.proxy import ReverseProxy
@@ -59,6 +69,22 @@ 

  

  init_auth(login_manager, conf.auth_backend)

  

+ # Set up telemetry exporter if configured.

+ provider = TracerProvider(resource=Resource.create({SERVICE_NAME: "cts"}))

+ trace.set_tracer_provider(provider)

+ exporter_url = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")

+ if exporter_url == "console":

+     provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))

+ elif exporter_url:

+     provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))

+ 

+ tracer = trace.get_tracer(__name__)

+ 

+ # Initialize Flask drop-in instrumentation middleware

+ FlaskInstrumentor().instrument_app(app, tracer_provider=provider)

+ if "CTS_INSTRUMENT_DATABASE" in os.environ:

+     SQLAlchemyInstrumentor().instrument()

+ 

  

  def json_error(status, error, message):

      response = jsonify({"status": status, "error": error, "message": message})

file modified
+7
@@ -27,6 +27,9 @@ 

  from logging import getLogger

  from sqlalchemy.orm import attributes

  

+ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator

+ 

+ 

  log = getLogger()

  

  _cache_lock = Lock()
@@ -74,6 +77,10 @@ 

                  extra_args["agent"] = flask.g.user.username

              else:

                  extra_args["agent"] = None

+             # Add telemetry information. This includes an extra key

+             # traceparent.

+             TraceContextTextMapPropagator().inject(extra_args)

+ 

              msg.update(extra_args)

              _cached_composes[comp.id].append(msg)

  

file modified
+3
@@ -12,3 +12,6 @@ 

  apispec==5.2.2  # Pin version to install from pypi

  apispec-webframeworks==0.5.2  # Pin version to install from pypi

  marshmallow

+ opentelemetry-instrumentation-flask

+ opentelemetry-instrumentation-sqlalchemy

+ opentelemetry-exporter-otlp-proto-http

There are two commits here.

The first one adds basic tracing to the API: each request would create a new span.

The second commit adds tracing exporter for sqlalchemy. That adds a span for each executed query.

If OTEL_EXPORTER_OTLP_ENDPOING is set to console, any other non-empty value configures the collector to send the traces to (similar to ODCS).

The messages sent to message bus get a new field traceparent with a trace ID.

rebased onto b33cea6d6178f1956d5d002e14c4bd10ed4418f5

10 months ago

I'm not sure if the database logging makes much sense. It uncovered an N+1 issue with the compose list endpoint, but overall I'm not sure how useful it could be.

rebased onto a268a452f875756827698c1d8305fbda6b9892ca

9 months ago

This is ready for review. The database tracing is included, by disabled by default and needs to be explicitly enabled.

2 new commits added

  • Add instrumentation for database queries
  • Add basic OpenTelemetry integration
9 months ago

New requires are not added to requirements.txt

rebased onto 6bd6dd0

9 months ago

Looks good to me. :thumbsup:

Pull-Request has been merged by lsedlar

9 months ago