#15 WIP: Updating for new buildbot
Closed 5 years ago by tflink. Opened 5 years ago by tflink.
taskotron/ tflink/execdb feature/maketestable  into  develop

file modified
+101 -89
@@ -28,95 +28,107 @@ 

  # the version as used in setup.py

  __version__ = "0.0.11"

  

- 

- # Flask App

- app = Flask(__name__)

- app.secret_key = 'replace-me-with-something-random'

- 

- 

- # Load default config, then override that with a config file

- if os.getenv('DEV') == 'true':

-     default_config_obj = 'execdb.config.DevelopmentConfig'

-     default_config_file = os.getcwd() + '/conf/settings.py'

- elif os.getenv('TEST') == 'true':

-     default_config_obj = 'execdb.config.TestingConfig'

-     default_config_file = os.getcwd() + '/conf/settings.py'

- else:

-     default_config_obj = 'execdb.config.ProductionConfig'

-     default_config_file = '/etc/execdb/settings.py'

- 

- app.config.from_object(default_config_obj)

- 

- config_file = os.environ.get('EXECDB_CONFIG', default_config_file)

- 

- if os.path.exists(config_file):

-     app.config.from_pyfile(config_file)

- 

- if app.config['PRODUCTION']:

-     if app.secret_key == 'replace-me-with-something-random':

-         raise Warning("You need to change the app.secret_key value for production")

- 

- # setup logging

- fmt = '[%(filename)s:%(lineno)d] ' if app.debug else '%(module)-12s '

- fmt += '%(asctime)s %(levelname)-7s %(message)s'

- datefmt = '%Y-%m-%d %H:%M:%S'

- loglevel = logging.DEBUG if app.debug else logging.INFO

- formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)

- 

- 

- def setup_logging():

-     root_logger = logging.getLogger('')

-     root_logger.setLevel(logging.DEBUG)

- 

-     if app.config['STREAM_LOGGING']:

-         print("doing stream logging")

-         stream_handler = logging.StreamHandler()

-         stream_handler.setLevel(loglevel)

-         stream_handler.setFormatter(formatter)

-         root_logger.addHandler(stream_handler)

-         app.logger.addHandler(stream_handler)

- 

-     if app.config['SYSLOG_LOGGING']:

-         print("doing syslog logging")

-         syslog_handler = logging.handlers.SysLogHandler(

-             address='/dev/log',

-             facility=logging.handlers.SysLogHandler.LOG_LOCAL4)

-         syslog_handler.setLevel(loglevel)

-         syslog_handler.setFormatter(formatter)

-         root_logger.addHandler(syslog_handler)

-         app.logger.addHandler(syslog_handler)

- 

-     if app.config['FILE_LOGGING'] and app.config['LOGFILE']:

-         print("doing file logging to %s" % app.config['LOGFILE'])

-         file_handler = logging.handlers.RotatingFileHandler(

-             app.config['LOGFILE'],

-             maxBytes=500000,

-             backupCount=5)

-         file_handler.setLevel(loglevel)

-         file_handler.setFormatter(formatter)

-         root_logger.addHandler(file_handler)

-         app.logger.addHandler(file_handler)

- 

- setup_logging()

- 

- if app.config['SHOW_DB_URI']:

-     app.logger.debug('using DBURI: %s' % app.config['SQLALCHEMY_DATABASE_URI'])

- 

- 

- # database

- db = SQLAlchemy(app)

- 

- # setup login manager

+ #db = SQLAlchemy()

  login_manager = LoginManager()

- login_manager.setup_app(app)

- login_manager.login_view = 'login_page.login'

- 

- # register blueprints

- from execdb.controllers.main import main

- app.register_blueprint(main)

  

- from execdb.controllers.login_page import login_page

- app.register_blueprint(login_page)

  

- from execdb.controllers.admin import admin

- app.register_blueprint(admin)

+ def create_app(test_config=None):

+ 

+     # Flask App

+     app = Flask(__name__)

+     app.secret_key = 'replace-me-with-something-random'

+ 

+     # Load default config, then override that with a config file

+     if os.getenv('DEV') == 'true':

+         default_config_obj = 'execdb.config.DevelopmentConfig'

+         default_config_file = os.getcwd() + '/conf/settings.py'

+     elif os.getenv('TEST') == 'true':

+         default_config_obj = 'execdb.config.TestingConfig'

+         default_config_file = os.getcwd() + '/conf/settings.py'

+     else:

+         default_config_obj = 'execdb.config.ProductionConfig'

+         default_config_file = '/etc/execdb/settings.py'

+ 

+     config_file = os.environ.get('EXECDB_CONFIG', default_config_file)

+ 

+     app.config.from_object(default_config_obj)

+ 

+     # only load config from file if we don't have a test config

+     if test_config is None:

+ 

+         if os.path.exists(config_file):

+             app.config.from_pyfile(config_file)

+ 

+         if app.config['PRODUCTION']:

+             if app.secret_key == 'replace-me-with-something-random':

+                 raise Warning("You need to change the app.secret_key value for production")

+     else:

+         app.config.update(test_config)

+ 

+ 

+     # setup logging

+     fmt = '[%(filename)s:%(lineno)d] ' if app.debug else '%(module)-12s '

+     fmt += '%(asctime)s %(levelname)-7s %(message)s'

+     datefmt = '%Y-%m-%d %H:%M:%S'

+     loglevel = logging.DEBUG if app.debug else logging.INFO

+     formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)

+ 

+     def setup_logging():

+         root_logger = logging.getLogger('')

+         root_logger.setLevel(logging.DEBUG)

+ 

+         if app.config['STREAM_LOGGING']:

+             print("doing stream logging")

+             stream_handler = logging.StreamHandler()

+             stream_handler.setLevel(loglevel)

+             stream_handler.setFormatter(formatter)

+             root_logger.addHandler(stream_handler)

+             app.logger.addHandler(stream_handler)

+ 

+         if app.config['SYSLOG_LOGGING']:

+             print("doing syslog logging")

+             syslog_handler = logging.handlers.SysLogHandler(

+                 address='/dev/log',

+                 facility=logging.handlers.SysLogHandler.LOG_LOCAL4)

+             syslog_handler.setLevel(loglevel)

+             syslog_handler.setFormatter(formatter)

+             root_logger.addHandler(syslog_handler)

+             app.logger.addHandler(syslog_handler)

+ 

+         if app.config['FILE_LOGGING'] and app.config['LOGFILE']:

+             print("doing file logging to %s" % app.config['LOGFILE'])

+             file_handler = logging.handlers.RotatingFileHandler(

+                 app.config['LOGFILE'],

+                 maxBytes=500000,

+                 backupCount=5)

+             file_handler.setLevel(loglevel)

+             file_handler.setFormatter(formatter)

+             root_logger.addHandler(file_handler)

+             app.logger.addHandler(file_handler)

+ 

+     setup_logging()

+ 

+     if app.config['SHOW_DB_URI']:

+         app.logger.debug('using DBURI: %s' % app.config['SQLALCHEMY_DATABASE_URI'])

+ 

+ 

+     # database

+     from execdb.models import db

+     db.init_app(app)

+     app.db = db

+ 

+     # setup login manager

+     login_manager.setup_app(app)

+     login_manager.login_view = 'login_page.login'

+ 

+     # register blueprints

+     from execdb.controllers.main import main

+     app.register_blueprint(main)

+ 

+     from execdb.controllers.login_page import login_page

+     app.register_blueprint(login_page)

+ 

+     from execdb.controllers.admin import admin

+     app.register_blueprint(admin)

+ 

+     return app

@@ -27,7 +27,8 @@ 

  from flask_login import login_user, logout_user, login_required, current_user, AnonymousUserMixin

  

  

- from execdb import app, login_manager

+ #from execdb import app, login_manager

+ from execdb import login_manager

  from execdb.models.user import User

  

  login_page = Blueprint('login_page', __name__)

file modified
+106 -31
@@ -17,7 +17,7 @@ 

  # Authors:

  #    Josef Skladanka <jskladan@redhat.com>

  

- from flask import Blueprint, render_template, request, jsonify

+ from flask import Blueprint, render_template, request, jsonify, current_app

  import werkzeug.exceptions

  from sqlalchemy.orm import exc as orm_exc

  
@@ -26,18 +26,21 @@ 

  from werkzeug.exceptions import BadRequest as JSONBadRequest

  

  

- from execdb import app, db

+ #from execdb import app, db

+ #from execdb import db

+ from execdb.models import db

  from execdb.models.job import Job, BuildStep

  from sqlalchemy import desc

  

  import json

  import re

+ from datetime import datetime

  

  from pprint import pformat

  

  main = Blueprint('main', __name__)

- BB_URL = app.config['BUILDBOT_FRONTPAGE_URL']

- RESULTSDB_URL = app.config['RESULTSDB_FRONTPAGE_URL']

+ #BB_URL = current_app.config['BUILDBOT_FRONTPAGE_URL']

+ #RESULTSDB_URL = current_app.config['RESULTSDB_FRONTPAGE_URL']

  

  RE_PAGE = re.compile(r"([?&])page=([0-9]+)")

  RP = {}
@@ -110,7 +113,8 @@ 

  

      return render_template('index.html',

                             jobs=jobs,

-                            buildbot_url=BB_URL,

+ #                           buildbot_url=BB_URL,

+                            buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'],

                             prev=prev,

                             next=next)

  
@@ -124,9 +128,11 @@ 

      job.t_triggered = str(job.t_triggered).split('.')[0]

      return render_template('show_job.html',

                             job=job,

-                            buildbot_url=BB_URL,

-                            resultsdb_url=RESULTSDB_URL,

-                            artifacts_base_url=app.config['ARTIFACTS_BASE_URL'])

+                            #buildbot_url=BB_URL,

+                            buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'],

+                            #resultsdb_url=RESULTSDB_URL,

+                            resultsdb_url=current_app.config['RESULTSDB_FRONTPAGE_URL'],

+                            artifacts_base_url=current_app.config['ARTIFACTS_BASE_URL'])

  

  

  @main.route('/jobs/<uuid>/steps', methods=['GET'])
@@ -137,7 +143,8 @@ 

          return 'UUID not found', 404

  

      steps = dict(

-         buildbot_url=BB_URL,

+         #buildbot_url=BB_URL,

+         buildbot_url=current_app.config['BUILDBOT_FRONTPAGE_URL'],

          steps=[],

          job_status=job.current_state,

          job_duration=str(job.build_took),
@@ -213,7 +220,7 @@ 

      if event not in known_events:

          # FIXME remove

          if 'uuid' in json.dumps(data):

-             app.logger.debug("UUID found in %s", event)

+             current_app.logger.debug("UUID found in %s", event)

  

          return 'Skipping event', 204

  
@@ -261,7 +268,7 @@ 

  #        app.logger.debug("%s: %s" % (uuid, data['payload']['build']['steps']))

  #        app.logger.debug("%s - Build Started" % uuid)

          for step_info in data['payload']['build']['steps']:

-             #            app.logger.debug("%s -- adding step %s"% (uuid, step_info['name']))

+             #            current_app.logger.debug("%s -- adding step %s"% (uuid, step_info['name']))

              step = BuildStep(name=step_info['name'])

              step.job = job

              db.session.add(step)
@@ -270,11 +277,11 @@ 

  

      elif event == 'stepStarted' and job.current_state == 'Running':

          step_info = data['payload']['step']

- #        app.logger.debug("%s - Step Started -  %s"% (uuid, step_info['name']))

+ #        current_app.logger.debug("%s - Step Started -  %s"% (uuid, step_info['name']))

          try:

              step = job.get_build_step(step_info['name'])

          except KeyError:

-             app.logger.debug("Job %s had missing step %s", job.uuid, step_info)

+             current_app.logger.debug("Job %s had missing step %s", job.uuid, step_info)

              step = BuildStep(name=step_info['name'])

              step.job = job

  
@@ -283,11 +290,11 @@ 

          step.data = json.dumps(data['payload'])  # FIXME - store sensible subset of data

          db.session.add(step)

          db.session.commit()

- #        app.logger.debug("%s - Step Started -  %s - written to db"% (uuid, step_info['name']))

+ #        current_app.logger.debug("%s - Step Started -  %s - written to db"% (uuid, step_info['name']))

  

      elif event == 'stepFinished' and job.current_state == 'Running':

          step_info = data['payload']['step']

- #        app.logger.debug("%s - Step Finished -  %s"% (uuid, step_info['name']))

+ #        current_app.logger.debug("%s - Step Finished -  %s"% (uuid, step_info['name']))

          try:

              step = job.get_build_step(step_info['name'])

          except KeyError:
@@ -303,36 +310,104 @@ 

  

          db.session.add(step)

          db.session.commit()

- #        app.logger.debug("%s - Step Finished -  %s - written to db" % (uuid, step_info['name']))

+ #        current_app.logger.debug("%s - Step Finished -  %s - written to db" % (uuid, step_info['name']))

  

      elif event == 'buildFinished' and job.current_state == 'Running':

          job.finish()

          db.session.add(job)

          db.session.commit()

- #        app.logger.debug("%s - Build Finished " % uuid)

+ #        current_app.logger.debug("%s - Build Finished " % uuid)

  

+ def process_bb_status(status_data):

+     # grab uuid, build state, properties

+     build_properties = status_data['properties']

+     uuid = build_properties['uuid'][0]

+     build_complete = status_data['complete']

  

- @main.route('/buildbottest', methods=['POST'])

+     current_app.logger.info("Processing data for job {} (complete: {})".format(uuid, build_complete))

+ 

+     # find job in db

+     try:

+         job = db.session.query(Job).filter(Job.uuid == uuid).one()

+     except orm_exc.NoResultFound:

+         current_app.logger.info("UUID {} not found".format(uuid))

+         return 'UUID not found', 400

+ 

+     # if 'complete' is false, create job

+     if not build_complete:

+         current_app.logger.debug("%s -- adding job %s"% (uuid, status_data['number']))

+ 

+         job.t_build_started = datetime.fromtimestamp(status_data['started_at'])

+         job.t_triggered = datetime.fromtimestamp(status_data['started_at'])

+ 

+         job.taskname = build_properties['taskname'][0]

+         job.item = build_properties['item'][0]

+         job.item_type = build_properties['item_type'][0]

+         job.arch = build_properties['arch'][0]

+         job.slavename = build_properties['slavename'][0]

+         job.link_build_log = '/builders/%s/builds/%s' % (

+             status_data['buildrequest']['builderid'],

+             status_data['number'])

+ 

+         db.session.add(job)

+ 

+         db.session.commit()

+ 

+     # if 'complete' is true, fill in buildsteps, finish job

+     else:

+         # add the completed time and state

+ 

+         job.t_build_ended = datetime.fromtimestamp(status_data['complete_at'])

+         # add the build steps

+         for step_info in status_data['steps']:

+ 

+             current_app.logger.debug("%s -- adding step %s"% (uuid, step_info['name']))

+ 

+             step = BuildStep(name=step_info['name'])

+             step.job = job

+             step.started_at = datetime.fromtimestamp(step_info['started_at'])

+             step.finished_at = datetime.fromtimestamp(step_info['complete_at'])

+             step.data = step_info['state_string']

+ 

+             # there doesn't seem to be a really reasonable way to tell if a step has failed but

+             # this should work well enough for now

+             if 'failed' in step_info['state_string']:

+                 step.status = 'NOT OK'

+             else:

+                 step.status = 'OK'

+ 

+             db.session.add(step)

+ 

+         db.session.commit()

+ 

+ 

+ @main.route('/buildbot', methods=['POST'])

  def bb_push():

      """

      Receives the post-push notifications from buildbot and fills in

      the steps for the job.

      """

-     # data are embedded in form field 'packets'

-     data = request.form

-     try:

-         data = request.form['packets']

-     except werkzeug.exceptions.BadRequestKeyError:

-         return 'Field `packets` missing in request form.', 400

-     data = json.loads(data)

- 

-     # app.logger.debug(pformat(data))

+     from pprint import pprint

  

-     # multiple messages may be present in one 'packet'

-     for entry in data:

-         process_event(entry)

- #        app.logger.debug("%s %s, %s", entry['id'], entry['event'], process_event(entry))

+     data = request.get_json()

+ #    pprint(data)

  

+     process_bb_status(data)

+     # data are embedded in form field 'packets'

+ #    data = request.form

+ #    try:

+ #        data = request.form['packets']

+ #    except werkzeug.exceptions.BadRequestKeyError:

+ #        return 'Field `packets` missing in request form.', 400

+ #    data = json.loads(data)

+ #

+ #    # current_app.logger.debug(pformat(data))

+ #

+ #    # multiple messages may be present in one 'packet'

+ #    for entry in data:

+ #        process_event(entry)

+ ##        current_app.logger.debug("%s %s, %s", entry['id'], entry['event'], process_event(entry))

+ #

      # plain 200 code needs to be returned - otherwise buildbot is

      # endlessly trying to re-send the message.

      # FIXME - add logging for non-200 responses

@@ -0,0 +1,2 @@ 

+ from flask_sqlalchemy import SQLAlchemy

+ db = SQLAlchemy()

file modified
+1 -1
@@ -17,7 +17,7 @@ 

  # Authors:

  #    Josef Skladanka <jskladan@redhat.com>

  

- from execdb import db, app

+ from . import db

  

  import datetime

  import uuid

file modified
+2 -1
@@ -17,7 +17,8 @@ 

  # Authors:

  #    Josef Skladanka <jskladan@redhat.com>

  

- from execdb import db

+ from . import db

+ 

  from flask_login import UserMixin

  from werkzeug.security import generate_password_hash, check_password_hash

  

file modified
+6
@@ -6,3 +6,9 @@ 

  

  [pep8]

  max-line-length=99

+ 

+ [pytest]

+ minversion=2.0

+ python_functions=test should

+ python_files=test_* functest_*

+ #addopts=--functional tests/ --cov-report=term-missing --cov execdb

This is a WIP for discussion purposes. It does work with the new buildbot, though.

I started off wanting to test the changes I was going to make which led me down a path of modifying the application structure to use a factory pattern but didn't question whether that was wise until later on.

As we don't know how much longer execdb is going to be a thing, does it make sense to be making changes to the application like this? It is more easily testable this way, though and could be a decent reference for new Flask apps going forward?

Either way, it still needs a bit of cleanup but I wanted to start the discussion before putting any more time into this

IIUIC, the only "benefit" of these changes isthe fact you can pass test_config dict to the create_app function, instead of setting TEST=true env in testing/conftest.py to load the Test config.

I might be missing something, but I don't really see the testability increase here. I might be just limited in my understanding of the benefits, by the scope of the apps we develop, but I'm not sure the (from my point of view) minor enhancements outweighs the gained complications it brings because of the current_app being available only at request, or app creation - like before/after request handling, access to app out of the request scope (I'm shooting a bit wild bere, but Alembic and cmdline interface come to mind).

IMO pruning the __init__.py, and/or making sure all the "options" are working properly, and sensible, would be of bigger benefit.

I could see the value of the factory pattern, if we wanted to serve multiple instances of the same app, but with different configs from a single web-server (but then again, I think we have bigger fish to fry with the way our apps are served in the first place).

So, what would be the actual benefits to this, compared to the way we develop/test the flask apps ATM? To me, it just seems like we would be replacing one "weird thing" in one place(setting the env variable in conftest.py), for multiple new ones scattered all over the code.

On top of that, I'm still not convinced that dropping execdb alltogether would not be the better option.

IIRC, it now serves a two-fold purpose:

  1. creating the UUID
  2. acting as a HUB through which an informed user can tie the 'result' in resultsdb with the 'build' in buildbot (by visiting the result's Group, and then clicking on logs link to get to the ExecDB interface, where the link to the Buildbot job is provided

UUID could easily be generated in trigger, easily (just replacing the ExecDB api call with the proper "give me an UUID" method from python's stdlib).

With the second point - I'm not sure anybody really uses it, but we could easily create a file in artifacts directory containing an url of the relevant build, and pointing the resultsdb's Group log-link to that file. Sure, this would mean a change in Taskotron (resultsdb_directive, specifically IMO), but it would be minor, and it IMO is tempting to just drop the whole ExecDB, when it's so easy to replace.

...in artifacts directory containing an url of the relevant build, and pointing the resultsdb's Group log-link to that file.

Or we could setup mod_rewrite to treat the file as a rewrite map, and the client could be redirected to the buildbot's build detail directly, replacing the current functionality in a seamless way: https://httpd.apache.org/docs/2.4/rewrite/rewritemap.html#txt (not sure it's worth the trouble, nor necessary, just wanted to show the possibility)

On top of that, I'm still not convinced that dropping execdb alltogether would not be the better option.

I just talked to @adamwill and he asked us whether we could support the new standard for CI messages in Taskotron:
https://pagure.io/fedora-ci/messages
That would mean we send a message when a task is created/scheduled, started executing, and finished executing. Currently we only support messages when the result is posted, IIRC (which also not always matches the input parameters of the scheduled task, e.g. rpmdeplint).
I'd wait with the implementation at least until we talk to Miro Vadkerti next week, but this might be a reason why ExecDB could still be useful for tracking these states (and possibly sending the messages). So this might need some additional thought before we decide to drop it or not. If there's an easy way to work with the new buildbot and still keep it, I guess that would be preferable for the moment (or wait for the next week's discussions).

Fair point, even though I still believe the ExecDB is not needed even for this, and the messages can as easily be sent from trigger and buildbot.
All in all, ExecDB, as is, IMO does not add any benefit, and only adds to the amount of stuff we need to keep running. It merelyre-stores the data already present in other places (mostly buildbot), while not adding significant amount of data on top of that.
With the changes in buildbot/execdb, it will only really store the "job was triggered, buildbot started, buildbot finished" information, since we will lose all the "steps" information (buildbot does not send push notifications on step changes any more).

If both of you are set on/prefer keeping execdb in place, I'm not the one to keep you from it. I'm merely trying to point out, that it was/is mostly irrelevant, and that I'm not sure investing any time into re-implementing it/keeping it alive makes sense.

Not much more I could add to that, I think.

We talked about this in person and decided to go with the easiest change possible so I'm closing this PR, will remove the refactoring and just change how execdb is handling incoming data

Pull-Request has been closed by tflink

5 years ago