#23 Add database support and dummy Flask frontend
Merged 8 years ago by jkaluza. Opened 8 years ago by jkaluza.
jkaluza/freshmaker db-support  into  master

file modified
+14
@@ -9,6 +9,15 @@ 

  

  

  class BaseConfiguration(object):

+     # Make this random (used to generate session keys)

+     SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0'

+     SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join(

+         dbdir, 'freshmaker.db'))

+     SQLALCHEMY_TRACK_MODIFICATIONS = False

+ 

+     HOST = '0.0.0.0'

+     PORT = 5001

+ 

      DEBUG = False

      # Global network-related values, in seconds

      NET_TIMEOUT = 120
@@ -63,6 +72,7 @@ 

      # Settings for docker image rebuild handler

      KOJI_CONTAINER_SCRATCH_BUILD = False

  

+     SSL_ENABLED = False

  

  class DevConfiguration(BaseConfiguration):

      DEBUG = True
@@ -82,6 +92,10 @@ 

      LOG_BACKEND = 'console'

      LOG_LEVEL = 'debug'

      DEBUG = True

+ 

+     SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(

+         path.join(dbdir, 'tests', 'test_freshmaker.db'))

+ 

      MESSAGING = 'in_memory'

      PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1'

  

file modified
+13 -1
@@ -24,9 +24,21 @@ 

  #            Jan Kaluza <jkaluza@redhat.com>

  

  from logging import getLogger

+ 

+ from flask import Flask

+ from flask_sqlalchemy import SQLAlchemy

+ 

  from freshmaker.logger import init_logging

  from freshmaker.config import init_config

+ from freshmaker.proxy import ReverseProxy

+ 

+ app = Flask(__name__)

+ app.wsgi_app = ReverseProxy(app.wsgi_app)

  

- conf = init_config()

+ db = SQLAlchemy(app)

+ 

+ conf = init_config(app)

  init_logging(conf)

  log = getLogger(__name__)

+ 

+ from freshmaker import views

file modified
+3 -2
@@ -31,7 +31,7 @@ 

  from freshmaker import logger

  

  

- def init_config():

+ def init_config(app):

      """

      Configure Freshmaker

      """
@@ -54,7 +54,7 @@ 

      if 'FRESHMAKER_CONFIG_SECTION' in os.environ:

          config_section = os.environ['FRESHMAKER_CONFIG_SECTION']

      # TestConfiguration shall only be used for running tests, otherwise...

-     if any(['py.test' in arg or 'pytest.py' in arg for arg in sys.argv]):

+     if any(['nosetests' in arg or 'noserunner.py' in arg or 'py.test' in arg or 'pytest.py' in arg for arg in sys.argv]):

          config_section = 'TestConfiguration'

          from conf import config

          config_module = config
@@ -81,6 +81,7 @@ 

      # finally configure Freshmaker

      config_section_obj = getattr(config_module, config_section)

      conf = Config(config_section_obj)

+     app.config.from_object(config_section_obj)

      return conf

  

  

file added
+166
@@ -0,0 +1,166 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2017  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ 

+ from flask_script import Manager

+ from functools import wraps

+ import flask_migrate

+ import logging

+ import os

+ import ssl

+ 

+ import fedmsg.config

+ import moksha.hub

+ import moksha.hub.hub

+ import moksha.hub.reactor

+ 

+ from freshmaker import app, conf, db

+ from freshmaker import models

+ 

+ 

+ manager = Manager(app)

+ help_args = ('-?', '--help')

+ manager.help_args = help_args

+ migrate = flask_migrate.Migrate(app, db)

+ manager.add_command('db', flask_migrate.MigrateCommand)

+ 

+ 

+ def console_script_help(f):

+     @wraps(f)

+     def wrapped(*args, **kwargs):

+         import sys

+         if any([arg in help_args for arg in sys.argv[1:]]):

+             command = os.path.basename(sys.argv[0])

+             print("""{0}

+ 

+ Usage: {0} [{1}]

+ 

+ See also:

+   freshmaker-manager(1)""".format(command,

+                            '|'.join(help_args)))

+             sys.exit(2)

+         r = f(*args, **kwargs)

+         return r

+     return wrapped

+ 

+ 

+ def _establish_ssl_context():

+     if not conf.ssl_enabled:

+         return None

+     # First, do some validation of the configuration

+     attributes = (

+         'ssl_certificate_file',

+         'ssl_certificate_key_file',

+         'ssl_ca_certificate_file',

+     )

+ 

+     for attribute in attributes:

+         value = getattr(conf, attribute, None)

+         if not value:

+             raise ValueError("%r could not be found" % attribute)

+         if not os.path.exists(value):

+             raise OSError("%s: %s file not found." % (attribute, value))

+ 

+     # Then, establish the ssl context and return it

+     ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)

+     ssl_ctx.load_cert_chain(conf.ssl_certificate_file,

+                             conf.ssl_certificate_key_file)

+     ssl_ctx.verify_mode = ssl.CERT_OPTIONAL

+     ssl_ctx.load_verify_locations(cafile=conf.ssl_ca_certificate_file)

+     return ssl_ctx

+ 

+ 

+ @console_script_help

+ @manager.command

+ def upgradedb():

+     """ Upgrades the database schema to the latest revision

+     """

+     app.config["SERVER_NAME"] = 'localhost'

+     migrations_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)),

+                                   'migrations')

+     with app.app_context():

+         flask_migrate.upgrade(directory=migrations_dir)

+ 

+ @console_script_help

+ @manager.command

+ def cleardb():

+     """ Clears the database

+     """

+     models.Event.query.delete()

+     models.ArtifactBuild.query.delete()

+     db.session.commit()

+ 

+ @manager.command

+ @console_script_help

+ def generatelocalhostcert():

+     """ Creates a public/private key pair for message signing and the frontend

+     """

+     from OpenSSL import crypto

+     cert_key = crypto.PKey()

+     cert_key.generate_key(crypto.TYPE_RSA, 2048)

+ 

+     with open(conf.ssl_certificate_key_file, 'w') as cert_key_file:

+         os.chmod(conf.ssl_certificate_key_file, 0o600)

+         cert_key_file.write(

+             crypto.dump_privatekey(crypto.FILETYPE_PEM, cert_key))

+ 

+     cert = crypto.X509()

+     msg_cert_subject = cert.get_subject()

+     msg_cert_subject.C = 'US'

+     msg_cert_subject.ST = 'MA'

+     msg_cert_subject.L = 'Boston'

+     msg_cert_subject.O = 'Development'

+     msg_cert_subject.CN = 'localhost'

+     cert.set_serial_number(2)

+     cert.gmtime_adj_notBefore(0)

+     cert.gmtime_adj_notAfter(315360000)  # 10 years

+     cert.set_issuer(cert.get_subject())

+     cert.set_pubkey(cert_key)

+     cert_extensions = [

+         crypto.X509Extension(

+             'keyUsage', True,

+             'digitalSignature, keyEncipherment, nonRepudiation'),

+         crypto.X509Extension('extendedKeyUsage', True, 'serverAuth'),

+     ]

+     cert.add_extensions(cert_extensions)

+     cert.sign(cert_key, 'sha256')

+ 

+     with open(conf.ssl_certificate_file, 'w') as cert_file:

+         cert_file.write(

+             crypto.dump_certificate(crypto.FILETYPE_PEM, cert))

+ 

+ 

+ @console_script_help

+ @manager.command

+ def runssl(host=conf.host, port=conf.port, debug=conf.debug):

+     """ Runs the Flask app with the HTTPS settings configured in config.py

+     """

+     logging.info('Starting Freshmaker frontend')

+ 

+     ssl_ctx = _establish_ssl_context()

+     app.run(

+         host=host,

+         port=port,

+         ssl_context=ssl_ctx,

+         debug=debug

+     )

+ 

+ if __name__ == "__main__":

+     manager.run()

@@ -0,0 +1,1 @@ 

+ Generic single-database configuration. 

\ No newline at end of file

@@ -0,0 +1,45 @@ 

+ # A generic, single database configuration.

+ 

+ [alembic]

+ # template used to generate migration files

+ # file_template = %%(rev)s_%%(slug)s

+ 

+ # set to 'true' to run the environment during

+ # the 'revision' command, regardless of autogenerate

+ # revision_environment = false

+ 

+ 

+ # Logging configuration

+ [loggers]

+ keys = root,sqlalchemy,alembic

+ 

+ [handlers]

+ keys = console

+ 

+ [formatters]

+ keys = generic

+ 

+ [logger_root]

+ level = WARN

+ handlers = console

+ qualname =

+ 

+ [logger_sqlalchemy]

+ level = WARN

+ handlers =

+ qualname = sqlalchemy.engine

+ 

+ [logger_alembic]

+ level = INFO

+ handlers =

+ qualname = alembic

+ 

+ [handler_console]

+ class = StreamHandler

+ args = (sys.stderr,)

+ level = NOTSET

+ formatter = generic

+ 

+ [formatter_generic]

+ format = %(levelname)-5.5s [%(name)s] %(message)s

+ datefmt = %H:%M:%S

@@ -0,0 +1,87 @@ 

+ from __future__ import with_statement

+ from alembic import context

+ from sqlalchemy import engine_from_config, pool

+ from logging.config import fileConfig

+ import logging

+ 

+ # this is the Alembic Config object, which provides

+ # access to the values within the .ini file in use.

+ config = context.config

+ 

+ # Interpret the config file for Python logging.

+ # This line sets up loggers basically.

+ fileConfig(config.config_file_name)

+ logger = logging.getLogger('alembic.env')

+ 

+ # add your model's MetaData object here

+ # for 'autogenerate' support

+ # from myapp import mymodel

+ # target_metadata = mymodel.Base.metadata

+ from flask import current_app

+ config.set_main_option('sqlalchemy.url',

+                        current_app.config.get('SQLALCHEMY_DATABASE_URI'))

+ target_metadata = current_app.extensions['migrate'].db.metadata

+ 

+ # other values from the config, defined by the needs of env.py,

+ # can be acquired:

+ # my_important_option = config.get_main_option("my_important_option")

+ # ... etc.

+ 

+ 

+ def run_migrations_offline():

+     """Run migrations in 'offline' mode.

+ 

+     This configures the context with just a URL

+     and not an Engine, though an Engine is acceptable

+     here as well.  By skipping the Engine creation

+     we don't even need a DBAPI to be available.

+ 

+     Calls to context.execute() here emit the given string to the

+     script output.

+ 

+     """

+     url = config.get_main_option("sqlalchemy.url")

+     context.configure(url=url)

+ 

+     with context.begin_transaction():

+         context.run_migrations()

+ 

+ 

+ def run_migrations_online():

+     """Run migrations in 'online' mode.

+ 

+     In this scenario we need to create an Engine

+     and associate a connection with the context.

+ 

+     """

+ 

+     # this callback is used to prevent an auto-migration from being generated

+     # when there are no changes to the schema

+     # reference: http://alembic.readthedocs.org/en/latest/cookbook.html

+     def process_revision_directives(context, revision, directives):

+         if getattr(config.cmd_opts, 'autogenerate', False):

+             script = directives[0]

+             if script.upgrade_ops.is_empty():

+                 directives[:] = []

+                 logger.info('No changes in schema detected.')

+ 

+     engine = engine_from_config(config.get_section(config.config_ini_section),

+                                 prefix='sqlalchemy.',

+                                 poolclass=pool.NullPool)

+ 

+     connection = engine.connect()

+     context.configure(connection=connection,

+                       target_metadata=target_metadata,

+                       process_revision_directives=process_revision_directives,

+                       **current_app.extensions['migrate'].configure_args)

+ 

+     try:

+         with context.begin_transaction():

+             context.run_migrations()

+     finally:

+         connection.close()

+ 

+ if context.is_offline_mode():

+     run_migrations_offline()

+ else:

+     run_migrations_online()

@@ -0,0 +1,22 @@ 

+ """${message}

+ 

+ Revision ID: ${up_revision}

+ Revises: ${down_revision}

+ Create Date: ${create_date}

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = ${repr(up_revision)}

+ down_revision = ${repr(down_revision)}

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ ${imports if imports else ""}

+ 

+ def upgrade():

+     ${upgrades if upgrades else "pass"}

+ 

+ 

+ def downgrade():

+     ${downgrades if downgrades else "pass"}

@@ -0,0 +1,41 @@ 

+ """Initial database

+ 

+ Revision ID: 1529069af28e

+ Revises: None

+ Create Date: 2017-04-28 13:43:23.340055

+ 

+ """

+ 

+ # revision identifiers, used by Alembic.

+ revision = '1529069af28e'

+ down_revision = None

+ 

+ from alembic import op

+ import sqlalchemy as sa

+ 

+ 

+ def upgrade():

+     op.create_table('events',

+     sa.Column('id', sa.Integer(), nullable=False),

+     sa.Column('message_id', sa.String(), nullable=False),

+     sa.PrimaryKeyConstraint('id')

+     )

+     op.create_table('artifact_builds',

+     sa.Column('id', sa.Integer(), nullable=False),

+     sa.Column('name', sa.String(), nullable=False),

+     sa.Column('type', sa.Integer(), nullable=True),

+     sa.Column('state', sa.Integer(), nullable=False),

+     sa.Column('time_submitted', sa.DateTime(), nullable=False),

+     sa.Column('time_completed', sa.DateTime(), nullable=True),

+     sa.Column('dep_of_id', sa.Integer(), nullable=True),

+     sa.Column('event_id', sa.Integer(), nullable=True),

+     sa.Column('build_id', sa.Integer(), nullable=True),

+     sa.ForeignKeyConstraint(['dep_of_id'], ['artifact_builds.id'], ),

+     sa.ForeignKeyConstraint(['event_id'], ['events.id'], ),

+     sa.PrimaryKeyConstraint('id')

+     )

+ 

+ 

+ def downgrade():

+     op.drop_table('artifact_builds')

+     op.drop_table('events')

file added
+134
@@ -0,0 +1,134 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2017  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ """ SQLAlchemy Database models for the Flask app

+ """

+ 

+ import contextlib

+ 

+ from datetime import datetime

+ from sqlalchemy import engine_from_config

+ from sqlalchemy.orm import (validates, scoped_session, sessionmaker,

+                             relationship)

+ from sqlalchemy.ext.declarative import declarative_base

+ 

+ from freshmaker import db, log

+ 

+ # BUILD_STATES for the builds submitted by Freshmaker

+ BUILD_STATES = {

+     # Artifact is building.

+     "build": 0,

+     # Artifact build is sucessfuly done.

+     "done": 1,

+     # Artifact build has failed.

+     "failed": 2,

+     # Artifact build is canceled.

+     "canceled": 3,

+ }

+ 

+ INVERSE_BUILD_STATES = {v: k for k, v in BUILD_STATES.items()}

+ 

+ ARTIFACT_TYPES = {

+     "rpm": 0,

+     "image": 1,

+     "module": 2,

+     }

+ 

+ INVERSE_ARTIFACT_TYPES = {v: k for k, v in ARTIFACT_TYPES.items()}

+ 

+ class FreshmakerBase(db.Model):

+     __abstract__ = True

+ 

+ class Event(FreshmakerBase):

+     __tablename__ = "events"

+     id = db.Column(db.Integer, primary_key=True)

+     message_id = db.Column(db.String, nullable=False)

+ 

+     # List of builds associated with this Event.

+     builds = relationship("ArtifactBuild", back_populates="event")

+ 

+     @classmethod

+     def create(cls, session, message_id):

+         event = cls(

+             message_id=message_id,

+         )

+         session.add(event)

+         return event

+ 

+ class ArtifactBuild(FreshmakerBase):

+     __tablename__ = "artifact_builds"

+     id = db.Column(db.Integer, primary_key=True)

+     name = db.Column(db.String, nullable=False)

+     type = db.Column(db.Integer)

+     state = db.Column(db.Integer, nullable=False)

+     time_submitted = db.Column(db.DateTime, nullable=False)

+     time_completed = db.Column(db.DateTime)

+ 

+     # Link to the Artifact on which this one depends and which triggered

+     # the rebuild of this Artifact.

+     dep_of_id = db.Column(db.Integer, db.ForeignKey('artifact_builds.id'))

+     dep_of = relationship('ArtifactBuild', remote_side=[id])

+ 

+     # Event associated with this Build

+     event_id = db.Column(db.Integer, db.ForeignKey('events.id'))

+     event = relationship("Event", back_populates="builds")

+ 

+     # Id of a build in the build system

+     build_id = db.Column(db.Integer)

+ 

+     @classmethod

+     def create(cls, session, event, name, type, build_id, dep_of=None):

+         now = datetime.utcnow()

+         build = cls(

+             name=name,

+             type=type,

+             event=event,

+             state="build",

+             build_id=build_id,

+             time_submitted=now,

+             dep_of=dep_of

+         )

+         session.add(build)

+         return build

+ 

+     @validates('state')

+     def validate_state(self, key, field):

+         if field in BUILD_STATES.values():

+             return field

+         if field in BUILD_STATES:

+             return BUILD_STATES[field]

+         raise ValueError("%s: %s, not in %r" % (key, field, BUILD_STATES))

+ 

+     @validates('type')

+     def validate_type(self, key, field):

+         if field in ARTIFACT_TYPES.values():

+             return field

+         if field in ARTIFACT_TYPES:

+             return ARTIFACT_TYPES[field]

+         raise ValueError("%s: %s, not in %r" % (key, field, ARTIFACT_TYPES))

+ 

+     def __repr__(self):

+         return "<ArtifactBuild %s, type %s, state %s, event %s>" % (

+             self.name, INVERSE_ARTIFACT_TYPES[self.type],

+             INVERSE_BUILD_STATES[self.state], self.event.message_id)

+ 

file added
+60
@@ -0,0 +1,60 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2017  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Ralph Bean <rbean@redhat.com>

+ 

+ """

+ Allows Freshmaker to run behind reverse proxy and to ensure redirects work

+ with https.

+ 

+ WSGI Middleware!!

+ 

+ Source: http://flask.pocoo.org/snippets/35/ by Peter Hansen

+ """

+ 

+ 

+ class ReverseProxy(object):

+     '''Wrap the application in this middleware and configure the

+     front-end server to add these headers, to let you quietly bind

+     this to a URL other than / and to an HTTP scheme that is

+     different than what is used locally.

+ 

+     :param app: the WSGI application

+     '''

+     def __init__(self, app):

+         self.app = app

+ 

+     def __call__(self, environ, start_response):

+         script_name = environ.get('HTTP_X_SCRIPT_NAME', '')

+         if script_name:

+             environ['SCRIPT_NAME'] = script_name

+             path_info = environ['PATH_INFO']

+             if path_info.startswith(script_name):

+                 environ['PATH_INFO'] = path_info[len(script_name):]

+ 

+         server = environ.get('HTTP_X_FORWARDED_HOST', '')

+         if server:

+             environ['HTTP_HOST'] = server

+ 

+         scheme = environ.get('HTTP_X_SCHEME', '')

+         if scheme:

+             environ['wsgi.url_scheme'] = scheme

+         return self.app(environ, start_response)

file added
+56
@@ -0,0 +1,56 @@ 

+ # -*- coding: utf-8 -*-

+ # Copyright (c) 2017  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import json

+ from flask import request, jsonify

+ from flask.views import MethodView

+ 

+ from freshmaker import app, conf, log

+ from freshmaker import models, db

+ 

+ api_v1 = {

+     'freshmaker': {

+         'url': '/freshmaker/1/events/',

+         'options': {

+             'defaults': {'id': None},

+             'methods': ['GET'],

+         }

+     },

+ }

+ 

+ 

+ class FreshmakerAPI(MethodView):

+ 

+     def get(self, id):

+         return "Done", 200

+ 

+ def register_api_v1():

+     """ Registers version 1 of MBS API. """

+     module_view = FreshmakerAPI.as_view('freshmaker')

+     for key, val in api_v1.items():

+         app.add_url_rule(val['url'],

+                          endpoint=key,

+                          view_func=module_view,

+                          **val['options'])

+ 

+ register_api_v1()

file modified
+4
@@ -16,3 +16,7 @@ 

  six

  sqlalchemy

  futures  # Python 2 only

+ Flask

+ Flask-Migrate

+ Flask-SQLAlchemy

+ Flask-Script

start_backend_from_here start_from_here
file renamed
file was moved with no change to the file
@@ -0,0 +1,3 @@ 

+ #!/bin/bash

+ 

+ FRESHMAKER_DEVELOPER_ENV=1 python freshmaker/manage.py runssl

file added
+62
@@ -0,0 +1,62 @@ 

+ # Copyright (c) 2016  Red Hat, Inc.

+ #

+ # Permission is hereby granted, free of charge, to any person obtaining a copy

+ # of this software and associated documentation files (the "Software"), to deal

+ # in the Software without restriction, including without limitation the rights

+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

+ # copies of the Software, and to permit persons to whom the Software is

+ # furnished to do so, subject to the following conditions:

+ #

+ # The above copyright notice and this permission notice shall be included in all

+ # copies or substantial portions of the Software.

+ #

+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE

+ # SOFTWARE.

+ #

+ # Written by Jan Kaluza <jkaluza@redhat.com>

+ 

+ import unittest

+ 

+ from freshmaker import conf, db

+ from freshmaker.models import Event, ArtifactBuild

+ 

+ 

+ class TestModels(unittest.TestCase):

+     def setUp(self):

+         db.session.remove()

+         db.drop_all()

+         db.create_all()

+         db.session.commit()

+ 

+     def tearDown(self):

+         db.session.remove()

+         db.drop_all()

+         db.session.commit()

+ 

+     def test_creating_event_and_builds(self):

+         event = Event.create(db.session, "test_msg_id")

+         build = ArtifactBuild.create(db.session, event, "ed", "module", 1234)

+         build2 = ArtifactBuild.create(db.session, event, "mksh", "module", 1235, build)

+         db.session.commit()

+         db.session.expire_all()

+ 

+         e = db.session.query(Event).filter(event.id == 1).one()

+         self.assertEquals(e.message_id, "test_msg_id")

+         self.assertEquals(len(e.builds), 2)

+ 

+         self.assertEquals(e.builds[0].name, "ed")

+         self.assertEquals(e.builds[0].type, 2)

+         self.assertEquals(e.builds[0].state, 0)

+         self.assertEquals(e.builds[0].build_id, 1234)

+         self.assertEquals(e.builds[0].dep_of, None)

+ 

+         self.assertEquals(e.builds[1].name, "mksh")

+         self.assertEquals(e.builds[1].type, 2)

+         self.assertEquals(e.builds[1].state, 0)

+         self.assertEquals(e.builds[1].build_id, 1235)

+         self.assertEquals(e.builds[1].dep_of.name, "ed")

The schema is following:

sqlite> .schema events
CREATE TABLE events (
        id INTEGER NOT NULL, 
        message_id VARCHAR NOT NULL, 
        PRIMARY KEY (id)
);
sqlite> .schema artifact_builds
CREATE TABLE artifact_builds (
        id INTEGER NOT NULL, 
        name VARCHAR NOT NULL, 
        type INTEGER, 
        state INTEGER NOT NULL, 
        time_submitted DATETIME NOT NULL, 
        time_completed DATETIME, 
        dep_of_id INTEGER, 
        event_id INTEGER, 
        build_id INTEGER, 
        PRIMARY KEY (id), 
        FOREIGN KEY(dep_of_id) REFERENCES artifact_builds (id), 
        FOREIGN KEY(event_id) REFERENCES events (id)
);

How it could work:

  • When we submit new build X to some build system (Koji, MBS, ...) triggered by fedmsg event E, we will just add the Event to "events" table and build to "artifact_builds".
  • We will listen on message bus to find out whether the build X has finished successfully. If not, we will just update the build state in "artifact_builds" to failed.
  • If the build of X succeeds we will update the state to "done" and if there are other artifacts depending on this artifact, we will submit their builds to build system and create new rows in artifact_builds with event_id se to E and dep_of set to artifact X.

That way we can track what artifacts have been rebuilt as a result of particular event or particular artifact rebuid/update. We can also use this data to check for circular dependencies.

I only saw one cosmetic typo. Otherwise, :+1: from me.

When call

with make_session(conf) as session:
    ArtifactBuild.create(session, ...)

session.commit will be called twice. Would this be a problem when use SQLAlchemy?

And, if call session.commit in create, it would prevent from creating multiple ArtifactBuild at once.

I guess this command should be wrapped with console_script_help as well.

It seems like you can drop this function and call manager.run() directly below.

Not sure about this. It seems like this is not being used anywhere. If you want to make flask-sqlalchemy automatically rollback the session if an exception happens, you probably need to do something like this:

@app.teardown_request
def teardown_request(exception):
if exception:
db.session.rollback()
db.session.remove()
db.session.remove()

at once

I meant in one transaction.

To be honest I have not checked this method deeply, it is copy-paste from the MBS code where we apparently use it in the poller thread which creates his own independent session. So far this is dead code for Freshmaker, so I'm going to remove this method.

rebased

8 years ago

Fixed the problems found during the review. Please re-review and I will ideally merge it :).

Pull-Request has been merged by jkaluza

8 years ago