#1571 Cleanup module_build_service/__init__.py
Closed 4 years ago by mprahl. Opened 4 years ago by cqi.
cqi/fm-orchestrator cleanup-mbs-init_py  into  v3

@@ -19,22 +19,14 @@ 

  """

  

  from __future__ import absolute_import

- from logging import getLogger

  

- from celery import Celery

- import gi  # noqa

- gi.require_version("Modulemd", "2.0")  # noqa

- from gi.repository import Modulemd  # noqa

+ import pkg_resources

+ 

  from flask import Flask, has_app_context, url_for

  from flask_sqlalchemy import SQLAlchemy

- import pkg_resources

  from sqlalchemy.pool import StaticPool

  

- from module_build_service.common.config import init_config

- from module_build_service.common.errors import (

-     ValidationError, Unauthorized, UnprocessableEntity, Conflict, NotFound,

-     Forbidden, json_error)

- from module_build_service.common.logger import init_logging, ModuleBuildLogs, level_flags, MBSLogger

+ from module_build_service.common.config import config_section

  from module_build_service.web.proxy import ReverseProxy

  

  try:
@@ -43,22 +35,10 @@ 

      version = "unknown"

  api_version = 2

  

- conf, config_section = init_config()

  app = Flask(__name__)

  app.wsgi_app = ReverseProxy(app.wsgi_app)

  app.config.from_object(config_section)

  

- celery_app = Celery("module-build-service")

- # Convert config names specific for Celery like this:

- # celery_broker_url -> broker_url

- celery_configs = {

-     name[7:]: getattr(conf, name)

-     for name in dir(conf) if name.startswith("celery_")

- }

- # Only allow a single process so that tasks are always serial per worker

- celery_configs["worker_concurrency"] = 1

- celery_app.conf.update(**celery_configs)

- 

  

  class MBSSQLAlchemy(SQLAlchemy):

      """
@@ -83,73 +63,6 @@ 

  db = MBSSQLAlchemy(app)

  

  

- def create_app(debug=False, verbose=False, quiet=False):

-     # logging (intended for flask-script, see manage.py)

-     log = getLogger(__name__)

-     if debug:

-         log.setLevel(level_flags["debug"])

-     elif verbose:

-         log.setLevel(level_flags["verbose"])

-     elif quiet:

-         log.setLevel(level_flags["quiet"])

- 

-     return app

- 

- 

- def load_views():

-     from module_build_service.web import views

- 

-     assert views

- 

- 

- @app.errorhandler(ValidationError)

- def validationerror_error(e):

-     """Flask error handler for ValidationError exceptions"""

-     return json_error(400, "Bad Request", str(e))

- 

- 

- @app.errorhandler(Unauthorized)

- def unauthorized_error(e):

-     """Flask error handler for NotAuthorized exceptions"""

-     return json_error(401, "Unauthorized", str(e))

- 

- 

- @app.errorhandler(Forbidden)

- def forbidden_error(e):

-     """Flask error handler for Forbidden exceptions"""

-     return json_error(403, "Forbidden", str(e))

- 

- 

- @app.errorhandler(RuntimeError)

- def runtimeerror_error(e):

-     """Flask error handler for RuntimeError exceptions"""

-     log.exception("RuntimeError exception raised")

-     return json_error(500, "Internal Server Error", str(e))

- 

- 

- @app.errorhandler(UnprocessableEntity)

- def unprocessableentity_error(e):

-     """Flask error handler for UnprocessableEntity exceptions"""

-     return json_error(422, "Unprocessable Entity", str(e))

- 

- 

- @app.errorhandler(Conflict)

- def conflict_error(e):

-     """Flask error handler for Conflict exceptions"""

-     return json_error(409, "Conflict", str(e))

- 

- 

- @app.errorhandler(NotFound)

- def notfound_error(e):

-     """Flask error handler for Conflict exceptions"""

-     return json_error(404, "Not Found", str(e))

- 

- 

- init_logging(conf)

- log = MBSLogger()

- build_logs = ModuleBuildLogs(conf.build_logs_dir, conf.build_logs_name_format, conf.log_level)

- 

- 

  def get_url_for(*args, **kwargs):

      """

      flask.url_for wrapper which creates the app_context on-the-fly.
@@ -161,6 +74,7 @@ 

      # system as the web views.

      app.config["SERVER_NAME"] = "localhost"

      with app.app_context():

+         from module_build_service.common import log

          log.debug(

              "WARNING: get_url_for() has been called without the Flask "

              "app_context. That can lead to SQLAlchemy errors caused by "
@@ -169,4 +83,10 @@ 

          return url_for(*args, **kwargs)

  

  

+ def load_views():

+     from module_build_service.web import views

+ 

+     assert views

+ 

+ 

  load_views()

@@ -21,7 +21,8 @@ 

  import pungi.arch

  from six import text_type

  

- from module_build_service import conf, log, build_logs, Modulemd

+ from module_build_service.common.modulemd import Modulemd

+ from module_build_service.common import conf, log, build_logs

  from module_build_service.common.koji import get_session, koji_retrying_multicall_map

  from module_build_service.common.scm import SCM

  from module_build_service.common.utils import load_mmd, mmd_to_str, to_text_type

@@ -22,11 +22,10 @@ 

  import kobo.rpmlib

  from OpenSSL.SSL import SysCallError

  

- from module_build_service import log, conf

  from module_build_service.builder import GenericBuilder

  from module_build_service.builder.KojiContentGenerator import KojiContentGenerator

  from module_build_service.builder.utils import execute_cmd, get_rpm_release, validate_koji_tag

- from module_build_service.common import models

+ from module_build_service.common import log, conf, models

  from module_build_service.common.koji import (

      get_session, koji_multicall_map, koji_retrying_multicall_map,

  )

@@ -13,7 +13,6 @@ 

  import kobo.rpmlib

  import platform

  

- from module_build_service import conf, log, Modulemd

  from module_build_service.builder import GenericBuilder

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

  from module_build_service.builder.utils import (
@@ -23,8 +22,9 @@ 

      get_koji_config,

      validate_koji_tag,

  )

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.koji import get_session

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import import_mmd, load_mmd_file, mmd_to_str

  from module_build_service.scheduler import events

  from module_build_service.scheduler.db_session import db_session

@@ -9,8 +9,7 @@ 

  from requests.exceptions import ConnectionError

  import six

  

- from module_build_service import conf, log

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.models import BUILD_STATES

  from module_build_service.common.retry import retry

  from module_build_service.resolver import GenericResolver

@@ -13,8 +13,7 @@ 

  import munch

  import requests

  

- from module_build_service import conf, log

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.errors import ValidationError, ProgrammingError

  

  

@@ -0,0 +1,12 @@ 

+ # -*- coding: utf-8 -*-

+ # SPDX-License-Identifier: MIT

+ 

+ from module_build_service.common.config import conf

+ from module_build_service.common import logger

+ 

+ __all__ = ('conf', 'log', 'build_logs')

+ 

+ logger.init_logging(conf)

+ log = logger.MBSLogger()

+ build_logs = logger.ModuleBuildLogs(

+     conf.build_logs_dir, conf.build_logs_name_format, conf.log_level)

@@ -986,3 +986,6 @@ 

          if i < 1:

              raise ValueError("NUM_THREADS_FOR_BUILD_SUBMISSIONS must be >= 1")

          self._num_threads_for_build_submissions = i

+ 

+ 

+ conf, config_section = init_config()

@@ -4,8 +4,6 @@ 

  

  from __future__ import absolute_import

  

- from flask import jsonify

- 

  

  class ValidationError(ValueError):

      pass
@@ -43,11 +41,5 @@ 

      pass

  

  

- def json_error(status, error, message):

-     response = jsonify({"status": status, "error": error, "message": message})

-     response.status_code = status

-     return response

- 

- 

  class IgnoreMessage(Exception):

      """Raise if message received from message bus should be ignored"""

@@ -7,7 +7,7 @@ 

  import munch

  import six.moves.xmlrpc_client as xmlrpclib

  

- from module_build_service import log

+ from module_build_service.common import log

  from module_build_service.common.retry import retry

  from module_build_service.common.errors import ProgrammingError

  

@@ -6,7 +6,7 @@ 

  

  import pkg_resources

  

- from module_build_service import conf, log

+ from module_build_service.common import conf, log

  from module_build_service.scheduler.parser import FedmsgMessageParser

  

  

@@ -17,7 +17,8 @@ 

  from sqlalchemy.orm import lazyload

  from sqlalchemy.orm import validates, load_only

  

- from module_build_service import db, log, get_url_for, conf

+ from module_build_service import db, get_url_for

+ from module_build_service.common import conf, log

  from module_build_service.common.errors import UnprocessableEntity

  import module_build_service.common.messaging

  from module_build_service.common.utils import load_mmd

@@ -0,0 +1,8 @@ 

+ # -*- coding: utf-8 -*-

+ # SPDX-License-Identifier: MIT

+ 

+ import gi

+ gi.require_version("Modulemd", "2.0")

+ from gi.repository import Modulemd  # noqa

+ 

+ __all__ = ['Modulemd']

@@ -2,7 +2,7 @@ 

  # SPDX-License-Identifier: MIT

  from __future__ import absolute_import

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common.errors import StreamAmbigous

  from module_build_service.resolver import GenericResolver

  

@@ -4,7 +4,7 @@ 

  import functools

  import time

  

- from module_build_service import conf, log

+ from module_build_service.common import conf, log

  

  

  def retry(timeout=conf.net_timeout, interval=conf.net_retry_interval, wait_on=Exception):

@@ -10,7 +10,7 @@ 

  import shutil

  import tempfile

  

- from module_build_service import log, conf

+ from module_build_service.common import log, conf

  from module_build_service.common.errors import (

      Forbidden,

      ValidationError,

@@ -7,7 +7,7 @@ 

  import requests

  

  import module_build_service.common.scm

- from module_build_service import conf, log

+ from module_build_service.common import conf, log

  from module_build_service.common.errors import ValidationError

  from module_build_service.common.utils import load_mmd_file

  

@@ -8,8 +8,9 @@ 

  from gi.repository.GLib import Error as ModuleMDError

  from six import string_types, text_type

  

- from module_build_service import conf, log, Modulemd

+ from module_build_service.common import conf, log

  from module_build_service.common.errors import UnprocessableEntity

+ from module_build_service.common.modulemd import Modulemd

  

  

  def to_text_type(s):

@@ -11,12 +11,13 @@ 

  from flask_script import Manager, prompt_bool

  from werkzeug.datastructures import FileStorage

  

- from module_build_service import app, conf, create_app, db

+ from module_build_service import app, db

  from module_build_service.builder.MockModuleBuilder import (

      import_builds_from_local_dnf_repos, load_local_builds

  )

- from module_build_service.common import models

+ from module_build_service.common import conf, models

  from module_build_service.common.errors import StreamAmbigous

+ from module_build_service.common.logger import level_flags

  from module_build_service.common.utils import load_mmd_file, import_mmd

  import module_build_service.scheduler.consumer

  from module_build_service.scheduler.db_session import db_session
@@ -24,6 +25,19 @@ 

  from module_build_service.web.submit import submit_module_build_from_yaml

  

  

+ def create_app(debug=False, verbose=False, quiet=False):

+     # logging (intended for flask-script, see manage.py)

+     log = logging.getLogger(__name__)

+     if debug:

+         log.setLevel(level_flags["debug"])

+     elif verbose:

+         log.setLevel(level_flags["verbose"])

+     elif quiet:

+         log.setLevel(level_flags["quiet"])

+ 

+     return app

+ 

+ 

  manager = Manager(create_app)

  help_args = ("-?", "--help")

  manager.help_args = help_args

@@ -14,8 +14,9 @@ 

  import sqlalchemy as sa

  

  # Data migration imports

- from module_build_service import Modulemd, conf

+ from module_build_service.common import conf

  from module_build_service.common.models import ModuleBuild

+ from module_build_service.common.modulemd import Modulemd

  

  # Data migration tables

  mb = sa.Table(

@@ -16,7 +16,8 @@ 

  from sqlalchemy.ext.declarative import declarative_base

  

  # Data migration imports

- from module_build_service import Modulemd, conf

+ from module_build_service.common import conf

+ from module_build_service.common.modulemd import Modulemd

  

  

  Base = declarative_base()

@@ -14,8 +14,8 @@ 

  import sqlalchemy as sa

  

  # Data migration imports

- from module_build_service import Modulemd

  import hashlib

+ from module_build_service.common.modulemd import Modulemd

  

  

  modulebuild = sa.Table(

@@ -14,7 +14,7 @@ 

  import sqlalchemy as sa

  

  # Data migration imports

- from module_build_service import Modulemd

+ from module_build_service.common.modulemd import Modulemd

  import hashlib

  import json

  from collections import OrderedDict

@@ -12,8 +12,8 @@ 

  

  from alembic import op

  import sqlalchemy as sa

- from module_build_service import Modulemd

  from module_build_service.common.models import ModuleBuild

+ from module_build_service.common.modulemd import Modulemd

  

  

  modulebuild = sa.Table(

@@ -14,7 +14,7 @@ 

  import sqlalchemy as sa

  

  # Data migration imports

- from module_build_service import Modulemd

+ from module_build_service.common.modulemd import Modulemd

  import hashlib

  import json

  from collections import OrderedDict

@@ -5,8 +5,7 @@ 

  import sqlalchemy

  from sqlalchemy.orm import aliased

  

- from module_build_service import log

- from module_build_service.common import models

+ from module_build_service.common import log, models

  from module_build_service.common.errors import UnprocessableEntity

  from module_build_service.common.utils import load_mmd

  from module_build_service.resolver.base import GenericResolver

@@ -3,8 +3,7 @@ 

  from __future__ import absolute_import

  from itertools import groupby

  

- from module_build_service import conf, log

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.koji import get_session, koji_multicall_map

  from module_build_service.resolver.DBResolver import DBResolver

  

@@ -7,8 +7,8 @@ 

  

  import kobo.rpmlib

  

- from module_build_service import conf

  from module_build_service.common import models

+ from module_build_service.common.config import conf

  from module_build_service.common.errors import UnprocessableEntity

  from module_build_service.common.request_utils import requests_session

  from module_build_service.common.utils import load_mmd, import_mmd

@@ -7,8 +7,7 @@ 

  from abc import ABCMeta, abstractmethod

  import six

  

- from module_build_service import conf

- import module_build_service.common.config as cfg

+ from module_build_service.common.config import conf, SUPPORTED_RESOLVERS

  

  

  class GenericResolver(six.with_metaclass(ABCMeta)):
@@ -16,7 +15,7 @@ 

      External Api for resolvers

      """

  

-     _resolvers = cfg.SUPPORTED_RESOLVERS

+     _resolvers = SUPPORTED_RESOLVERS

  

      # Resolver name. Each subclass of GenericResolver must set its own name.

      backend = "generic"

@@ -1,3 +1,18 @@ 

  # -*- coding: utf-8 -*-

  # SPDX-License-Identifier: MIT

  """ This is a sub-module for backend/scheduler functionality. """

+ 

+ from celery import Celery

+ 

+ from module_build_service.common.config import conf

+ 

+ celery_app = Celery("module-build-service")

+ # Convert config names specific for Celery like this:

+ # celery_broker_url -> broker_url

+ celery_configs = {

+     name[7:]: getattr(conf, name)

+     for name in dir(conf) if name.startswith("celery_")

+ }

+ # Only allow a single process so that tasks are always serial per worker

+ celery_configs["worker_concurrency"] = 1

+ celery_app.conf.update(**celery_configs)

@@ -4,8 +4,7 @@ 

  import concurrent.futures

  import threading

  

- from module_build_service import conf, log

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.scheduler import events

  from module_build_service.scheduler.db_session import db_session

  from module_build_service.scheduler.reuse import get_reusable_components, reuse_component

@@ -20,8 +20,7 @@ 

  import moksha.hub

  import sqlalchemy.exc

  

- from module_build_service import log, conf

- from module_build_service.common import models

+ from module_build_service.common import log, conf, models

  from module_build_service.common.errors import IgnoreMessage

  import module_build_service.common.messaging

  from module_build_service.common.messaging import default_messaging_backend

@@ -6,7 +6,7 @@ 

  from sqlalchemy.pool import NullPool

  from sqlalchemy.orm import scoped_session, sessionmaker

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common.models import session_before_commit_handlers

  

  __all__ = ("db_session",)

@@ -11,10 +11,10 @@ 

  import koji

  import six.moves.xmlrpc_client as xmlrpclib

  

- from module_build_service import conf, log, Modulemd

- from module_build_service.common import models, scm

+ from module_build_service.common import conf, log, models, scm

  from module_build_service.common.errors import UnprocessableEntity

  from module_build_service.common.koji import get_session, koji_retrying_multicall_map

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.resolve import (

      expand_single_mse_streams, get_compatible_base_module_mmds

  )

@@ -17,7 +17,7 @@ 

  import sched

  import time

  

- from module_build_service import log

+ from module_build_service.common import log

  

  

  KOJI_BUILD_CHANGE = "koji_build_change"

@@ -6,7 +6,7 @@ 

  

  import requests

  

- from module_build_service import log, conf

+ from module_build_service.common import log, conf

  from module_build_service.common.errors import GreenwaveError

  

  

@@ -7,12 +7,11 @@ 

  

  import koji

  

- from module_build_service import celery_app, conf, log

  from module_build_service.builder import GenericBuilder

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.koji import get_session

  from module_build_service.common.utils import mmd_to_str

- from module_build_service.scheduler import events

+ from module_build_service.scheduler import celery_app, events

  from module_build_service.scheduler.batches import continue_batch_build

  from module_build_service.scheduler.db_session import db_session

  

@@ -2,11 +2,11 @@ 

  # SPDX-License-Identifier: MIT

  from __future__ import absolute_import

  

- from module_build_service import celery_app, conf, log

+ from module_build_service.common import conf, log

  from module_build_service.common.koji import get_session

  from module_build_service.common.models import ModuleBuild, BUILD_STATES

  from module_build_service.scheduler.db_session import db_session

- from module_build_service.scheduler import events

+ from module_build_service.scheduler import celery_app, events

  

  

  def get_corresponding_module_build(nvr):

@@ -12,11 +12,11 @@ 

  from requests.exceptions import ConnectionError

  import six.moves.xmlrpc_client as xmlrpclib

  

- from module_build_service import celery_app, conf, log, build_logs

  from module_build_service.builder import GenericBuilder

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

  from module_build_service.builder.utils import get_rpm_release

  from module_build_service.common import models

+ from module_build_service.common import build_logs, conf, log

  from module_build_service.common.errors import UnprocessableEntity, Forbidden, ValidationError

  from module_build_service.common.utils import mmd_to_str

  from module_build_service.common.retry import retry
@@ -26,7 +26,7 @@ 

      record_filtered_rpms,

      record_module_build_arches

  )

- from module_build_service.scheduler import events

+ from module_build_service.scheduler import celery_app, events

  from module_build_service.scheduler.db_session import db_session

  from module_build_service.scheduler.default_modules import (

      add_default_modules, handle_collisions_with_base_module_rpms)

@@ -6,10 +6,9 @@ 

  from datetime import datetime

  import logging

  

- from module_build_service import celery_app, conf, log

  from module_build_service.builder import GenericBuilder

- from module_build_service.common import models

- from module_build_service.scheduler import events

+ from module_build_service.common import conf, log, models

+ from module_build_service.scheduler import celery_app, events

  from module_build_service.scheduler.batches import start_next_batch_build

  from module_build_service.scheduler.db_session import db_session

  

@@ -7,10 +7,9 @@ 

  

  import koji

  

- from module_build_service import celery_app, conf, log

  from module_build_service.builder import GenericBuilder

- from module_build_service.common import models

- from module_build_service.scheduler import events

+ from module_build_service.common import conf, log, models

+ from module_build_service.scheduler import celery_app, events

  from module_build_service.scheduler.db_session import db_session

  

  logging.basicConfig(level=logging.DEBUG)

@@ -3,7 +3,7 @@ 

  from __future__ import absolute_import

  import re

  

- from module_build_service import log

+ from module_build_service.common import log

  from module_build_service.common.errors import IgnoreMessage

  from module_build_service.scheduler import events

  

@@ -8,12 +8,12 @@ 

  from sqlalchemy.orm import lazyload, load_only

  

  

- from module_build_service import celery_app, conf, log

+ from module_build_service.common import conf, log, models

  from module_build_service.builder import GenericBuilder

- from module_build_service.common import models

  from module_build_service.common.koji import get_session

  import module_build_service.scheduler

  import module_build_service.scheduler.consumer

+ from module_build_service.scheduler import celery_app

  from module_build_service.scheduler.consumer import ON_MODULE_CHANGE_HANDLERS

  from module_build_service.scheduler.batches import (

      at_concurrent_component_threshold,

@@ -4,8 +4,7 @@ 

  

  import kobo.rpmlib

  

- from module_build_service import log, conf

- from module_build_service.common import models

+ from module_build_service.common import log, conf, models

  from module_build_service.common.resolve import get_base_module_mmds

  from module_build_service.resolver import GenericResolver

  from module_build_service.scheduler import events

@@ -5,8 +5,7 @@ 

  from __future__ import absolute_import

  import inspect

  

- from module_build_service import conf, log

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.scheduler.db_session import db_session

  from module_build_service.scheduler.handlers.greenwave import get_corresponding_module_build

  

@@ -8,10 +8,10 @@ 

  

  import kobo.rpmlib

  

- from module_build_service import conf, log, Modulemd

- from module_build_service.common import models

- from module_build_service.common.errors import ValidationError, UnprocessableEntity, Forbidden

  import module_build_service.common.scm

+ from module_build_service.common import conf, log, models

+ from module_build_service.common.errors import ValidationError, UnprocessableEntity, Forbidden

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.submit import fetch_mmd

  from module_build_service.common.utils import to_text_type

  from module_build_service.scheduler.db_session import db_session

@@ -3,7 +3,7 @@ 

  from __future__ import absolute_import

  import re

  

- from module_build_service import conf, log

+ from module_build_service.common import conf, log

  from module_build_service.common.koji import get_session

  from module_build_service.resolver import GenericResolver

  from module_build_service.scheduler.db_session import db_session

@@ -10,7 +10,8 @@ 

  from flask import g

  import requests

  

- from module_build_service import app, log, conf

+ from module_build_service import app

+ from module_build_service.common import conf, log

  from module_build_service.common.errors import Unauthorized, Forbidden

  

  

@@ -5,7 +5,7 @@ 

  from flask import request

  from flask.json import dumps

  

- import module_build_service

+ from module_build_service import app

  

  

  def jsonify(*args, **kwargs):
@@ -16,7 +16,7 @@ 

      indent = None

      separators = (",", ":")

  

-     if module_build_service.app.config["JSONIFY_PRETTYPRINT_REGULAR"] and not request.is_xhr:

+     if app.config["JSONIFY_PRETTYPRINT_REGULAR"] and not request.is_xhr:

          indent = 2

          separators = (", ", ": ")

  
@@ -31,6 +31,6 @@ 

  

      # Note that we add '\n' to end of response

      # (see https://github.com/mitsuhiko/flask/pull/1262)

-     rv = module_build_service.app.response_class(

+     rv = app.response_class(

          (dumps(data, indent=indent, separators=separators), "\n"), mimetype="application/json")

      return rv

@@ -6,8 +6,7 @@ 

  

  import solv

  

- from module_build_service import log, conf

- from module_build_service.common.models import ModuleBuild

+ from module_build_service.common import log, conf, models

  

  

  class MMDResolver(object):
@@ -144,7 +143,8 @@ 

                          # In case x.y.z versioning is not used for this base module, do not

                          # use versions solv.Dep.

                          stream_version_str = str(

-                             ModuleBuild.get_stream_version(stream_for_version, right_pad=False))

+                             models.ModuleBuild.get_stream_version(

+                                 stream_for_version, right_pad=False))

                          if len(stream_version_str) < 5:

                              req_pos = rel_or_dep(

                                  req_pos, solv.REL_OR, self.module_dep(name, stream))
@@ -173,7 +173,7 @@ 

                              op = solv.REL_EQ

                              if not exact_versions:

                                  op |= solv.REL_GT

-                             version = ModuleBuild.get_stream_version(

+                             version = models.ModuleBuild.get_stream_version(

                                  stream_for_version, right_pad=False

                              )

                              req_pos = rel_or_dep(
@@ -223,7 +223,8 @@ 

          # and so on. We therefore need to convert the stream and version of base module to

          # integer representation and add "module($name:$stream) = $stream_based_version"

          # to Provides.

-         stream_version = ModuleBuild.get_stream_version(mmd.get_stream_name(), right_pad=False)

+         stream_version = models.ModuleBuild.get_stream_version(

+             mmd.get_stream_name(), right_pad=False)

          if stream_version:

              base_stream_ver = True

              self.solvable_provides(

@@ -2,9 +2,9 @@ 

  # SPDX-License-Identifier: MIT

  from __future__ import absolute_import

  

- from module_build_service import log, Modulemd, conf

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.errors import StreamAmbigous, UnprocessableEntity

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.resolve import expand_single_mse_streams, get_base_module_mmds

  from module_build_service.common.utils import mmd_to_str

  from module_build_service.resolver import GenericResolver

@@ -11,9 +11,9 @@ 

  from gi.repository import GLib

  import requests

  

- from module_build_service import conf, log, Modulemd

- from module_build_service.common import models

+ from module_build_service.common import conf, log, models

  from module_build_service.common.errors import Conflict, Forbidden, ValidationError

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.submit import fetch_mmd

  from module_build_service.common.utils import load_mmd, mmd_to_str, to_text_type

  from module_build_service.web.mse import generate_expanded_mmds

@@ -11,8 +11,8 @@ 

  from sqlalchemy.orm import aliased

  from sqlalchemy.sql.sqltypes import Boolean as sqlalchemy_boolean

  

- from module_build_service import api_version, conf, db

- from module_build_service.common import models

+ from module_build_service import api_version, db

+ from module_build_service.common import conf, models

  from module_build_service.common.errors import ValidationError, NotFound

  from module_build_service.common.scm import scm_url_schemes

  

@@ -13,10 +13,11 @@ 

  from prometheus_client import generate_latest, CONTENT_TYPE_LATEST

  from six import string_types

  

- from module_build_service import app, conf, log, db, version, api_version as max_api_version

- from module_build_service.common import models

+ from module_build_service import app, db, version, api_version as max_api_version

+ from module_build_service.common import conf, log, models

  from module_build_service.common.errors import (

-     ValidationError, Forbidden, NotFound, ProgrammingError

+     ValidationError, Forbidden, NotFound, ProgrammingError,

+     Unauthorized, UnprocessableEntity, Conflict

  )

  from module_build_service.common.monitor import registry

  from module_build_service.common.submit import fetch_mmd
@@ -565,3 +566,52 @@ 

  

  

  register_api()

+ 

+ 

+ def json_error(status, error, message):

+     response = jsonify({"status": status, "error": error, "message": message})

+     response.status_code = status

+     return response

+ 

+ 

+ @app.errorhandler(ValidationError)

+ def validationerror_error(e):

+     """Flask error handler for ValidationError exceptions"""

+     return json_error(400, "Bad Request", str(e))

+ 

+ 

+ @app.errorhandler(Unauthorized)

+ def unauthorized_error(e):

+     """Flask error handler for NotAuthorized exceptions"""

+     return json_error(401, "Unauthorized", str(e))

+ 

+ 

+ @app.errorhandler(Forbidden)

+ def forbidden_error(e):

+     """Flask error handler for Forbidden exceptions"""

+     return json_error(403, "Forbidden", str(e))

+ 

+ 

+ @app.errorhandler(RuntimeError)

+ def runtimeerror_error(e):

+     """Flask error handler for RuntimeError exceptions"""

+     log.exception("RuntimeError exception raised")

+     return json_error(500, "Internal Server Error", str(e))

+ 

+ 

+ @app.errorhandler(UnprocessableEntity)

+ def unprocessableentity_error(e):

+     """Flask error handler for UnprocessableEntity exceptions"""

+     return json_error(422, "Unprocessable Entity", str(e))

+ 

+ 

+ @app.errorhandler(Conflict)

+ def conflict_error(e):

+     """Flask error handler for Conflict exceptions"""

+     return json_error(409, "Conflict", str(e))

+ 

+ 

+ @app.errorhandler(NotFound)

+ def notfound_error(e):

+     """Flask error handler for Conflict exceptions"""

+     return json_error(404, "Not Found", str(e))

file modified
+2 -1
@@ -14,7 +14,7 @@ 

  from six import string_types

  

  import module_build_service

- from module_build_service import db, Modulemd

+ from module_build_service import db

  from module_build_service.builder.utils import get_rpm_release

  from module_build_service.common.models import (

      BUILD_STATES,
@@ -23,6 +23,7 @@ 

      ModuleBuild,

      VirtualStream,

  )

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import load_mmd, import_mmd, mmd_to_str, to_text_type

  from module_build_service.scheduler.db_session import db_session

  

file modified
+2 -2
@@ -390,10 +390,10 @@ 

      """

  

      def _cleanup_build_logs():

-         build_ids = list(module_build_service.build_logs.handlers.keys())

+         build_ids = list(module_build_service.common.build_logs.handlers.keys())

          for build_id in build_ids:

              mock_build = mock.Mock()

              mock_build.id = build_id

-             module_build_service.build_logs.stop(mock_build)

+             module_build_service.common.build_logs.stop(mock_build)

  

      request.addfinalizer(_cleanup_build_logs)

@@ -21,12 +21,12 @@ 

  import pytest

  from werkzeug.datastructures import FileStorage

  

- from module_build_service import app, conf, build_logs, log

+ from module_build_service import app

  from module_build_service.builder import GenericBuilder

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

  from module_build_service.builder.MockModuleBuilder import load_local_builds

  from module_build_service.builder.utils import get_rpm_release, validate_koji_tag

- from module_build_service.common import models

+ from module_build_service.common import conf, log, build_logs, models

  from module_build_service.common.errors import Forbidden

  from module_build_service.common.utils import load_mmd, import_mmd

  from module_build_service.scheduler import events

@@ -7,9 +7,9 @@ 

  from mock import call, MagicMock, Mock, patch, PropertyMock

  import pytest

  

- from module_build_service import conf

  from module_build_service.builder import utils

  from module_build_service.common import models

+ from module_build_service.common.config import conf

  from module_build_service.common.errors import ProgrammingError, ValidationError

  from module_build_service.common.utils import load_mmd, import_mmd, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

@@ -11,9 +11,9 @@ 

  from mock import patch, Mock, call, mock_open

  import pytest

  

- from module_build_service import conf, build_logs, Modulemd

  from module_build_service.builder.KojiContentGenerator import KojiContentGenerator

- from module_build_service.common import models

+ from module_build_service.common import conf, build_logs, models

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

  from tests import init_data

@@ -11,10 +11,11 @@ 

  from mock import patch, MagicMock

  import pytest

  

- from module_build_service import conf, Modulemd

  from module_build_service.builder import GenericBuilder

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

  import module_build_service.common.models

+ from module_build_service.common.config import conf

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import mmd_to_str

  from module_build_service.scheduler import events

  from module_build_service.scheduler.db_session import db_session

@@ -11,7 +11,7 @@ 

  import mock

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.builder.MockModuleBuilder import (

      import_fake_base_module,

      import_builds_from_local_dnf_repos,
@@ -122,7 +122,7 @@ 

  

          return module

  

-     @mock.patch("module_build_service.conf.system", new="mock")

+     @mock.patch("module_build_service.common.conf.system", new="mock")

      def test_createrepo_filter_last_batch(self):

          module = self._create_module_with_filters(db_session, 3, koji.BUILD_STATES["COMPLETE"])

  
@@ -149,7 +149,7 @@ 

              rpm_names = [kobo.rpmlib.parse_nvr(rpm)["name"] for rpm in pkglist.split("\n")]

              assert "ed" not in rpm_names

  

-     @mock.patch("module_build_service.conf.system", new="mock")

+     @mock.patch("module_build_service.common.conf.system", new="mock")

      def test_createrepo_not_last_batch(self):

          module = self._create_module_with_filters(db_session, 2, koji.BUILD_STATES["COMPLETE"])

  
@@ -174,7 +174,7 @@ 

              rpm_names = [kobo.rpmlib.parse_nvr(rpm)["name"] for rpm in pkglist.split("\n")]

              assert "ed" in rpm_names

  

-     @mock.patch("module_build_service.conf.system", new="mock")

+     @mock.patch("module_build_service.common.conf.system", new="mock")

      def test_createrepo_empty_rmp_list(self):

          module = self._create_module_with_filters(db_session, 3, koji.BUILD_STATES["COMPLETE"])

  
@@ -194,7 +194,7 @@ 

      def setup_method(self, test_method):

          clean_database(add_platform_module=False)

  

-     @mock.patch("module_build_service.conf.system", new="mock")

+     @mock.patch("module_build_service.common.conf.system", new="mock")

      @mock.patch(

          "module_build_service.common.config.Config.base_module_repofiles",

          new_callable=mock.PropertyMock,

@@ -3,7 +3,7 @@ 

  from __future__ import absolute_import

  import os.path

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  

  

  class TestConfig:

@@ -6,8 +6,7 @@ 

  import shutil

  import tempfile

  

- from module_build_service import log

- from module_build_service.common import models

+ from module_build_service.common import log, models

  from module_build_service.common.logger import ModuleBuildLogs

  from module_build_service.scheduler.consumer import MBSConsumer

  from module_build_service.scheduler.db_session import db_session

@@ -5,7 +5,7 @@ 

  from mock import patch

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common.models import ComponentBuild, ComponentBuildTrace, ModuleBuild

  from module_build_service.common.utils import load_mmd, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

@@ -8,8 +8,8 @@ 

  import requests

  from six.moves import reload_module

  

- from module_build_service import app, conf

- from module_build_service.common import models

+ from module_build_service import app

+ from module_build_service.common import conf, models

  import module_build_service.common.monitor

  from module_build_service.scheduler.db_session import db_session

  from tests import init_data, make_module_in_db

@@ -5,8 +5,8 @@ 

  from mock import patch, PropertyMock

  import pytest

  

- from module_build_service import Modulemd

  from module_build_service.common import models

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import load_mmd

  from module_build_service.common.resolve import get_base_module_mmds

  from module_build_service.scheduler.db_session import db_session

@@ -7,11 +7,12 @@ 

  from mock import patch, PropertyMock

  import pytest

  

- from module_build_service import conf, Modulemd

  from module_build_service.builder.MockModuleBuilder import load_local_builds

  from module_build_service.common import models

+ from module_build_service.common.config import conf

  from module_build_service.common.errors import UnprocessableEntity

  from module_build_service.common.models import ModuleBuild

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str

  import module_build_service.resolver as mbs_resolver

  from module_build_service.scheduler.db_session import db_session

@@ -6,7 +6,7 @@ 

  from mock import patch, MagicMock

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common.models import ModuleBuild, BUILD_STATES

  from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str

  import module_build_service.resolver as mbs_resolver

@@ -5,7 +5,7 @@ 

  

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common.models import ModuleBuild

  from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

@@ -4,9 +4,10 @@ 

  

  from mock import patch, PropertyMock, Mock, call

  

- from module_build_service import app, conf

+ from module_build_service import app

  from module_build_service.builder.MockModuleBuilder import load_local_builds

  import module_build_service.common.models

+ from module_build_service.common import conf

  from module_build_service.common.utils import load_mmd, mmd_to_str

  import module_build_service.resolver as mbs_resolver

  from module_build_service.scheduler.db_session import db_session

@@ -7,7 +7,7 @@ 

  from mock import patch

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.builder import GenericBuilder

  from module_build_service.builder.KojiModuleBuilder import KojiModuleBuilder

  from module_build_service.builder.utils import validate_koji_tag

@@ -4,7 +4,8 @@ 

  

  import mock

  

- from module_build_service import celery_app, conf

+ from module_build_service.common.config import conf

+ from module_build_service.scheduler import celery_app

  from module_build_service.scheduler.handlers import components, greenwave, modules, repos, tags

  from module_build_service.scheduler.producer import fail_lost_builds

  from tests import scheduler_init_data

@@ -8,7 +8,7 @@ 

  from mock import call, Mock, patch, PropertyMock

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common.errors import UnprocessableEntity

  from module_build_service.common.models import ModuleBuild

  from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str

@@ -7,7 +7,7 @@ 

  from sqlalchemy import func

  

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  import module_build_service.common.config

  from module_build_service.common.models import BUILD_STATES, ModuleBuild

  from module_build_service.scheduler.consumer import MBSConsumer

@@ -5,7 +5,7 @@ 

  

  from mock import patch, PropertyMock

  

- from module_build_service import build_logs, conf

+ from module_build_service.common import build_logs, conf

  from module_build_service.common.models import ModuleBuild

  from module_build_service.common.utils import load_mmd, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

@@ -8,8 +8,9 @@ 

  from mock import patch

  import pytest

  

- from module_build_service import build_logs, conf, Modulemd

+ from module_build_service.common import build_logs, conf

  from module_build_service.common.models import ComponentBuild, ModuleBuild

+ from module_build_service.common.modulemd import Modulemd

  import module_build_service.resolver

  from module_build_service.scheduler.db_session import db_session

  import module_build_service.scheduler.handlers.modules

@@ -9,7 +9,7 @@ 

  from mock import call, patch

  import pytest

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.common import models

  from module_build_service.scheduler import producer

  from module_build_service.scheduler.db_session import db_session

@@ -6,8 +6,8 @@ 

  import pytest

  from sqlalchemy.orm.session import make_transient

  

- from module_build_service import Modulemd

  from module_build_service.common import models

+ from module_build_service.common.modulemd import Modulemd

  from module_build_service.common.utils import import_mmd, load_mmd, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

  from module_build_service.scheduler.reuse import get_reusable_component, get_reusable_module

@@ -6,8 +6,8 @@ 

  import mock

  import pytest

  

- from module_build_service import app, conf

- from module_build_service.common import models

+ from module_build_service import app

+ from module_build_service.common import conf, models

  from module_build_service.common.errors import UnprocessableEntity

  from module_build_service.common.utils import load_mmd, load_mmd_file, mmd_to_str

  from module_build_service.scheduler.db_session import db_session

@@ -4,7 +4,7 @@ 

  

  from mock import patch, Mock

  

- from module_build_service import conf

+ from module_build_service.common.config import conf

  from module_build_service.scheduler import ursine

  from tests import make_module, make_module_in_db, clean_database

  

Most of the code are moved to dedicated subpackages, but some others
can't due to the cycle dependencies.

Signed-off-by: Chenxiong Qi cqi@redhat.com

Build #743 failed (commit: 950365b87335c2be26fba284cc8b5e8d1fe07f81).
Rebase or make new commits to rebuild.

While preparing this PR, I found it is difficult to separate db and app into individual subpackages, my original thought is the web should own app and common/models.py should own db. But, it looks impossible due to the dependencies from db to app. That also means based on current implementation, it is impossible to separate flask_sqlalchemy from common/models.py, then the backend.

pretty please pagure-ci rebuild

4 years ago

@cqi I had filed FACTORY-5751 to look into using plain SQLAlchemy in common/models.py rather than Flask-SQLAlchemy. Here is a GitHub issue that I found that proposes a solution:
https://github.com/pallets/flask-sqlalchemy/issues/98

Once that is done, I think we can then fully untangle __init__.py

rebased onto e1303ad

4 years ago

I just rebased this PR and fixed a conflict.

@mprahl the info you mentioned is really great. Do you think this PR is acceptable as one step to the untaggled __init__.py, or we will put it together with the fix of FACTORY-5751?

Build #779 failed (commit: e1303ad).
Rebase or make new commits to rebuild.

Build #780 failed (commit: e1303ad).
Rebase or make new commits to rebuild.

@cqi my suggestion was not a requirement for this to get merged. Unfortunately, I won't have time to review this for a couple of weeks.

@mikem and @breilly could you please work with @cqi to review this? Please note that this is in the v3 branch, so it has a fairly different structure than what is currently in master.

@cqi thanks! I rebased it on the v3 branch and merged it in commit abae3c72acbb4286592c1fe49fee12aa06272c81.

Pull-Request has been closed by mprahl

4 years ago
Changes Summary 80
+10 -90
file changed
module_build_service/__init__.py
+2 -1
file changed
module_build_service/builder/KojiContentGenerator.py
+1 -2
file changed
module_build_service/builder/KojiModuleBuilder.py
+2 -2
file changed
module_build_service/builder/MockModuleBuilder.py
+1 -2
file changed
module_build_service/builder/base.py
+1 -2
file changed
module_build_service/builder/utils.py
+12 -0
file changed
module_build_service/common/__init__.py
+3 -0
file changed
module_build_service/common/config.py
+0 -8
file changed
module_build_service/common/errors.py
+1 -1
file changed
module_build_service/common/koji.py
+1 -1
file changed
module_build_service/common/messaging.py
+2 -1
file changed
module_build_service/common/models.py
+8
file added
module_build_service/common/modulemd.py
+1 -1
file changed
module_build_service/common/resolve.py
+1 -1
file changed
module_build_service/common/retry.py
+1 -1
file changed
module_build_service/common/scm.py
+1 -1
file changed
module_build_service/common/submit.py
+2 -1
file changed
module_build_service/common/utils.py
+16 -2
file changed
module_build_service/manage.py
+2 -1
file changed
module_build_service/migrations/versions/526fb7d445f7_module_buildrequires.py
+2 -1
file changed
module_build_service/migrations/versions/6d503efcd2b8_virtual_streams_table.py
+1 -1
file changed
module_build_service/migrations/versions/708ac8950f55_set_from_mmd_context.py
+1 -1
file changed
module_build_service/migrations/versions/9ca1c166f426_contexts.py
+1 -1
file changed
module_build_service/migrations/versions/a87264eeb49f_.py
+1 -1
file changed
module_build_service/migrations/versions/caeae7a4f537_ref_build_context.py
+1 -2
file changed
module_build_service/resolver/DBResolver.py
+1 -2
file changed
module_build_service/resolver/KojiResolver.py
+1 -1
file changed
module_build_service/resolver/MBSResolver.py
+2 -3
file changed
module_build_service/resolver/base.py
+15 -0
file changed
module_build_service/scheduler/__init__.py
+1 -2
file changed
module_build_service/scheduler/batches.py
+1 -2
file changed
module_build_service/scheduler/consumer.py
+1 -1
file changed
module_build_service/scheduler/db_session.py
+2 -2
file changed
module_build_service/scheduler/default_modules.py
+1 -1
file changed
module_build_service/scheduler/events.py
+1 -1
file changed
module_build_service/scheduler/greenwave.py
+2 -3
file changed
module_build_service/scheduler/handlers/components.py
+2 -2
file changed
module_build_service/scheduler/handlers/greenwave.py
+2 -2
file changed
module_build_service/scheduler/handlers/modules.py
+2 -3
file changed
module_build_service/scheduler/handlers/repos.py
+2 -3
file changed
module_build_service/scheduler/handlers/tags.py
+1 -1
file changed
module_build_service/scheduler/parser.py
+2 -2
file changed
module_build_service/scheduler/producer.py
+1 -2
file changed
module_build_service/scheduler/reuse.py
+1 -2
file changed
module_build_service/scheduler/route.py
+3 -3
file changed
module_build_service/scheduler/submit.py
+1 -1
file changed
module_build_service/scheduler/ursine.py
+2 -1
file changed
module_build_service/web/auth.py
+3 -3
file changed
module_build_service/web/backports.py
+6 -5
file changed
module_build_service/web/mmd_resolver.py
+2 -2
file changed
module_build_service/web/mse.py
+2 -2
file changed
module_build_service/web/submit.py
+2 -2
file changed
module_build_service/web/utils.py
+53 -3
file changed
module_build_service/web/views.py
+2 -1
file changed
tests/__init__.py
+2 -2
file changed
tests/conftest.py
+2 -2
file changed
tests/test_build/test_build.py
+1 -1
file changed
tests/test_builder/test_builder_utils.py
+2 -2
file changed
tests/test_builder/test_content_generator.py
+2 -1
file changed
tests/test_builder/test_koji.py
+5 -5
file changed
tests/test_builder/test_mock.py
+1 -1
file changed
tests/test_common/test_config.py
+1 -2
file changed
tests/test_common/test_logger.py
+1 -1
file changed
tests/test_common/test_models/test_models.py
+2 -2
file changed
tests/test_common/test_monitor.py
+1 -1
file changed
tests/test_common/test_resolve.py
+2 -1
file changed
tests/test_resolver/test_db.py
+1 -1
file changed
tests/test_resolver/test_koji.py
+1 -1
file changed
tests/test_resolver/test_local.py
+2 -1
file changed
tests/test_resolver/test_mbs.py
+1 -1
file changed
tests/test_scheduler/test_batches.py
+2 -1
file changed
tests/test_scheduler/test_celery_route_task.py
+1 -1
file changed
tests/test_scheduler/test_default_modules.py
+1 -1
file changed
tests/test_scheduler/test_greenwave_handler.py
+1 -1
file changed
tests/test_scheduler/test_module_init.py
+2 -1
file changed
tests/test_scheduler/test_module_wait.py
+1 -1
file changed
tests/test_scheduler/test_poller.py
+1 -1
file changed
tests/test_scheduler/test_reuse.py
+2 -2
file changed
tests/test_scheduler/test_submit.py
+1 -1
file changed
tests/test_scheduler/test_ursine.py