#90 Use flask and gunicorn instead of aiohttp.
Closed 5 years ago by cverna. Opened 5 years ago by cverna.
cverna/mdapi move_to_flask  into  master

file modified
+3 -2
@@ -6,9 +6,10 @@ 

  

  EXPOSE 8080

  

- RUN dnf -y install python3-aiohttp python3-werkzeug python3-requests python3-sqlalchemy python3-fedora-messaging

+ RUN dnf -y install python3-flask python3-requests python3-sqlalchemy python3-fedora-messaging python3-gunicorn

  

  USER 1001

  ENV MDAPI_CONFIG=/etc/mdapi/mdapi.cfg

+ ENV PYTHONPATH=/code

  COPY . /code

- ENTRYPOINT ["/code/mdapi-run"]

+ ENTRYPOINT ["gunicorn-3", "-w", "4", "-b", "0.0.0.0:8080", "--log-level=debug", "mdapi:app"]

file modified
+7 -20
@@ -14,23 +14,12 @@ 

  If you wish to set up a

  development instance of this project, follow these steps:

  

- * Install virtualenvwrapper

+ * Create and activate the virtualenv

  

  ::

  

-     dnf install python-virtualenvwrapper python3-devel openssl-devel

- 

- * Get the mkvirtualenv shell function

- 

- ::

- 

-     . /etc/profile.d/virtualenvwrapper.sh

- 

- * Create the virtualenv

- 

- ::

- 

-     mkvirtualenv mdapi -p python3

+     python3 -m venv .venv

+     source .venv/bin/activate

  

  * Install the dependencies

  
@@ -42,19 +31,17 @@ 

  

  ::

  

-     ./mdapi-get_repo_md mdapi/default_config.py

+     ./mdapi-get_repo_md mdapi/mdapi.cfg

  

  * Start the server

  

  ::

  

-     ./mdapi-run

- 

- * If you need to reactivate the virtual env later, use workon:

+     gunicorn-3 mdapi:app

  

+ * Run the test suite

  ::

  

-     workon mdapi

- 

+     tox

  

  **Note:** This project is python3 only

file modified
+25 -8
@@ -49,12 +49,13 @@ 

  

  import requests

  

- from sqlalchemy import text

+ import sqlalchemy as sa

+ from sqlalchemy.orm import sessionmaker

+ from sqlalchemy.orm import scoped_session

+ 

  from fedora_messaging.api import Message, publish

  from fedora_messaging.exceptions import PublishReturned, ConnectionException

  

- import mdapi.lib as mdapilib

- 

  KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos/'

  PKGDB2_URL = 'https://admin.fedoraproject.org/pkgdb/'

  DL_SERVER = 'http://dl.fedoraproject.org'
@@ -149,6 +150,22 @@ 

  }

  

  

+ @contextlib.contextmanager

+ def session_manager(db_url, debug=False, pool_recycle=3600):

+     """ A handy context manager for our sessions. """

+     engine = sa.create_engine(

+         db_url, echo=debug, pool_recycle=pool_recycle)

+     session = scoped_session(sessionmaker(bind=engine))

+     try:

+         yield session

+         session.commit()

+     except:

+         session.rollback()

+         raise

+     finally:

+         session.close()

+ 

+ 

  def list_branches(status='Active'):

      ''' Return the list of Fedora branches corresponding to the given

      status.
@@ -199,7 +216,7 @@ 

      print('%s Comparing %s and %s' % (name.ljust(padding), db1, db2))

  

      def get_table_names(uri):

-         with mdapilib.session_manager('sqlite:///' + uri) as session:

+         with session_manager('sqlite:///' + uri) as session:

              for name in session.connection().engine.table_names():

                  if name == 'db_info':

                      continue
@@ -213,8 +230,8 @@ 

          return name.split('(')[0]

  

      def get_all_rows(uri, table, cache):

-         query = text(queries.get(table, default_query).format(table=table))

-         with mdapilib.session_manager('sqlite:///' + uri) as session:

+         query = sa.text(queries.get(table, default_query).format(table=table))

+         with session_manager('sqlite:///' + uri) as session:

              engine = session.connection().engine

              for i, row in enumerate(engine.execute(query)):

                  if table in cache_dependant_tables:
@@ -232,8 +249,8 @@ 

  

  

      def build_cache(uri, cache):

-         query = text(packages_cache_builder.format(table=table))

-         with mdapilib.session_manager('sqlite:///' + uri) as session:

+         query = sa.text(packages_cache_builder.format(table=table))

+         with session_manager('sqlite:///' + uri) as session:

              engine = session.connection().engine

              for pkgId, pkgname in engine.execute(query):

                  cache[pkgId] = pkgname

file removed
-5
@@ -1,5 +0,0 @@ 

- #!/usr/bin/env python3

- 

- from mdapi.server import main

- 

- main()

file removed
-12
@@ -1,12 +0,0 @@ 

- [Unit]

- Description=mdapi, a simple API to expose metadata from RPM repositories

- Documentation=https://pagure.io/mdapi

- 

- [Service]

- Environment=MDAPI_CONFIG=/etc/mdapi/mdapi.cfg

- ExecStart=/usr/bin/mdapi-run

- Type=simple

- Restart=on-failure

- 

- [Install]

- WantedBy=multi-user.target

file removed
-202
@@ -1,202 +0,0 @@ 

- Name:           mdapi

- Version:        2.10.5

- Release:        1%{?dist}

- Summary:        A simple API to query the metadata of the repositories

- 

- License:        GPLv2+

- URL:            https://pagure.io/mdapi

- Source0:        https://pagure.io/releases/mdapi/%{name}-%{version}.tar.gz

- 

- BuildArch:      noarch

- 

- BuildRequires:  python3-aiohttp

- BuildRequires:  python3-requests

- BuildRequires:  python3-setuptools

- BuildRequires:  python3-sqlalchemy

- BuildRequires:  python3-werkzeug

- BuildRequires:  python3-devel

- BuildRequires:  systemd

- 

- Requires:  python3-aiohttp

- Requires:  python3-multidict

- Requires:  python3-requests

- Requires:  python3-setuptools

- Requires:  python3-sqlalchemy

- Requires:  python3-werkzeug

- 

- Requires(post):     systemd

- Requires(preun):    systemd

- Requires(postun):   systemd

- 

- 

- %description

- Small web and asynchronous application serving the metadata of the Fedora

- repositories

- 

- 

- %prep

- %setup -q

- 

- %build

- %{__python3} setup.py build

- 

- 

- %install

- %{__python3} setup.py install -O1 --skip-build --root=%{buildroot}

- 

- # Install the systemd service file

- mkdir -p $RPM_BUILD_ROOT/%{_unitdir}

- install -m 644 mdapi.service $RPM_BUILD_ROOT/%{_unitdir}/mdapi.service

- 

- 

- %post

- %systemd_post mdapi.service

- 

- %preun

- %systemd_preun mdapi.service

- 

- %postun

- %systemd_postun_with_restart mdapi.service

- 

- 

- %files

- %doc COPYING

- %{python3_sitelib}/mdapi/

- %{python3_sitelib}/mdapi*.egg-info

- %{_bindir}/mdapi-get_repo_md

- %{_bindir}/mdapi-run

- %{_unitdir}/mdapi.service

- 

- 

- %changelog

- * Fri Feb 09 2018 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.10.5-1

- - Update to 2.10.5

- - Fix the srcpkg endpoint by making it search for a direct match before trying

-   the sub-packages

- 

- * Fri Feb 09 2018 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.10.4-1

- - Update to 2.10.4

- - Fix the content_type header on JSONP output

- 

- * Wed Jan 10 2018 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.10.3-1

- - Update to 2.10.3

- - Always initialize the output variable

- - Fix getting package by their source name

- 

- * Tue Aug 08 2017 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.10.2-1

- - Update to 2.10.2

- - Fix importing MultiDict from multidict instead of aiohttp

- 

- * Tue Aug 08 2017 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.10.1-1

- - Update to 2.10.1

- - Fix the srcpkg endpoints

- 

- * Wed Aug 02 2017 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.10-1

- - Update to 2.10

- - Fix displaying the proper basename

- - Expose the upstream URL as present in the metadata in the JSON returned

- - Fix the JSONP support of the branches endpoint

- 

- * Wed Jul 12 2017 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.9-1

- - Update to 2.9

- - Add JSONP support to the application for easier integration with other apps

- 

- * Tue May 02 2017 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.8-1

- - Update to 2.8

- - Fix and improve the README

- - Set the content_type and charset of the index page

- - Download the source repository metadata as src_<name> and expose them

- - Order the list of branches returned

- 

- * Mon Sep 12 2016 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.7-1

- - Update to 2.7

- - Add the possibility to query package by the name of their source package

- - Return flags (Igor Gnatenko)

- - Fix querying packages by their source package name (Igor Gnatenko)

- 

- * Thu May 12 2016 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.6-1

- - Update to 2.6

- - Fix get_repo_md to handle gzip correctly (Patrick Uiterwijk)

- - Let the mdapi web app to return JSON with the correct mimetype (Patrick

-   Uiterwijk)

- - Adjust get_repo_md for the new URL structure on the mirrors

- 

- * Wed Mar 02 2016 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.5-1

- - Update to 2.5

- - Chain the method as coroutines making the process more asynchronous and a

-   little bit faster

- - Streamline the fedmsg diff publication to reduce the amount of data sent per

-   message (especially for new repo where the diff concerns all the packages)

- 

- * Mon Feb 29 2016 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.4-1

- - Update to 2.4

- - Adjust copyright year

- - Expand mdapi to return dependencies information (conflicts, enhances,

-   recommends, suggests, supplements, requires, provides and obsoletes)

- - Improve the cron script to retry after 30 seconds waiting, 3 times max if it

-   fails to get info for a repo

- 

- * Wed Jan 13 2016 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.3-1

- - Update to 2.3

- - Return pretty JSON when the accept header is text/html (Till Maas)

- - Order packages by their Epoch-Version-Release and return only the most recent

- - Mention deployment URL in the README

- - Adjust the JSON key from `files` to `changelogs` to reflect the content

- - Rely on urllib and aiohttp.MultiDict to do the url arguments parsing

- - Fix using ?pretty=False in a browser

- - Drop the dependency on simplejson

- 

- * Tue Nov 24 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.2.3-1

- - Update to 2.2.3

- - Fix the branches endpoint (un-instantiated variable)

- 

- * Mon Nov 23 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.2.2-1

- - Update to 2.2.2

- - Fix accessing the configuration to adjust the link on the front page

- 

- * Sun Nov 22 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.2.1-1

- - Update to 2.2.1

- - Fix the links in the front page with it's accessed without trailing slash

-   (Patrick Uiterwijk)

- 

- * Thu Nov 19 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.2-1

- - Update to 2.2

- - Fix typo in the cron job

- 

- * Thu Nov 19 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.1-1

- - Update to 2.1

- - Drop un-used import

- - Fix prettifying the JSON only on demand

- 

- * Thu Nov 19 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 2.0-1

- - Update to 2.0

- - Fix typos on the front page

- - Add the dependencies (Requires, Obsoletes, Provides, Conflicts, Enhances,

-   Recommends, Suggests, Supplements) to the JSON returned

- - Add the option to return a prettier JSON

- - Add the possibility to disable checking the SSL cert

- 

- * Tue Nov 10 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 1.2.2-1

- - Update to 1.2.2

- - Be consistent about providing links on the front page

- 

- * Tue Nov 10 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 1.2.1-1

- - Update to 1.2.1

- - Fix typo in the routing

- 

- * Tue Nov 10 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 1.2-1

- - Update 1.2

- - Add the possibility to prefix the URLs at which mdapi answers

- - Simplify the routing by listing them

- - Fix the example and link in the documentation on the front page

- 

- * Tue Nov 10 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 1.1-1

- - Update to 1.1

- - Make the URLs to pkgdb, the koji repo and the download server configurable for

-   the cron job

- 

- * Mon Nov 09 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 1.0-1

- - Update to 1.0

- 

- * Tue Oct 27 2015 Pierre-Yves Chibon <pingou@pingoured.fr> - 0.1-1

- - First package for Fedora

file modified
+81 -262
@@ -22,86 +22,36 @@ 

  '''

  Top level of the mdapi aiohttp application.

  '''

- import functools

- import json

- import logging

  import os

+ import logging

  

- import asyncio

- import werkzeug

- from aiohttp import web

+ from flask import Flask, jsonify, send_from_directory, abort

  

  import mdapi.lib as mdapilib

  

  

- CONFIG = dict()

- obj = werkzeug.import_string('mdapi.default_config')

- for key in dir(obj):

-     if key.isupper():

-         CONFIG[key] = getattr(obj, key)

- 

- 

- if 'MDAPI_CONFIG' in os.environ and os.path.exists(os.environ['MDAPI_CONFIG']):

-     with open(os.environ['MDAPI_CONFIG']) as config_file:

-         exec(compile(

-             config_file.read(), os.environ['MDAPI_CONFIG'], 'exec'), CONFIG)

- 

- indexfile = os.path.join(

-     os.path.dirname(os.path.abspath(__file__)), 'index.html')

- INDEX = ''

- with open(indexfile) as stream:

-     INDEX = stream.read()

-     INDEX = INDEX.replace('$PREFIX', CONFIG.get('PREFIX', ''))

- 

- 

- _log = logging.getLogger(__name__)

- 

- 

- def allows_jsonp(function):

-     ''' Add support for JSONP queries to the endpoint decorated. '''

+ app = Flask(__name__)

+ app.config["DB_FOLDER"] = "/var/tmp"

  

-     @functools.wraps(function)

-     def wrapper(request, *args, **kwargs):

-         ''' Actually does the job with the arguments provided.

+ gunicorn_logger = logging.getLogger("gunicorn.error")

+ app.logger.handlers = gunicorn_logger.handlers

+ app.logger.setLevel(gunicorn_logger.level)

  

-         :arg request: the request that was called that we want to add JSONP

-         support to

-         :type request: aiohttp.web_request.Request

  

-         '''

-         response = yield from function(request, *args, **kwargs)

-         url_arg = request.query

-         callback = url_arg.get('callback')

-         if callback and request.method == 'GET':

-             if isinstance(callback, list):

-                 callback = callback[0]

-             response.mimetype = 'application/javascript'

-             response.content_type = 'application/javascript'

-             response.text = '%s(%s);' % (callback, response.text)

- 

-         return response

- 

-     return wrapper

- 

- 

- @asyncio.coroutine

  def _get_pkg(branch, name=None, action=None, srcname=None):

      ''' Return the pkg information for the given package in the specified

      branch or raise an aiohttp exception.

      '''

-     if (not name and not srcname) or (name and srcname):

-         raise web.HTTPBadRequest()

- 

      pkg = None

      wrongdb = False

      for repotype in ['updates-testing', 'updates', 'testing', None]:

  

          if repotype:

              dbfile = '%s/mdapi-%s-%s-primary.sqlite' % (

-                 CONFIG['DB_FOLDER'], branch, repotype)

+                 app.config['DB_FOLDER'], branch, repotype)

          else:

              dbfile = '%s/mdapi-%s-primary.sqlite' % (

-                 CONFIG['DB_FOLDER'], branch)

+                 app.config['DB_FOLDER'], branch)

  

          if not os.path.exists(dbfile):

              wrongdb = True
@@ -109,41 +59,29 @@ 

  

          wrongdb = False

  

-         session = yield from mdapilib.create_session(

+         session = mdapilib.create_session(

              'sqlite:///%s' % dbfile)

          if name:

              if action:

-                 pkg = yield from mdapilib.get_package_by(

+                 pkg = mdapilib.get_package_by(

                      session, action, name)

              else:

-                 pkg = yield from mdapilib.get_package(session, name)

+                 pkg = mdapilib.get_package(session, name)

          elif srcname:

-             pkg = yield from mdapilib.get_package_by_src(session, srcname)

+             pkg = mdapilib.get_package_by_src(session, srcname)

          session.close()

          if pkg:

              break

  

      if wrongdb:

-         raise web.HTTPBadRequest()

+         abort(400)

  

      if not pkg:

-         raise web.HTTPNotFound()

+         abort(404)

  

      return (pkg, repotype)

  

  

- def _get_pretty(request):

-     pretty = False

-     params = request.query

-     if params.get('pretty') in ['1', 'true']:

-         pretty = True

-     # Assume pretty if html is requested and pretty is not disabled

-     elif 'text/html' in request.headers.get('ACCEPT', ''):

-         pretty = True

-     return pretty

- 

- 

- @asyncio.coroutine

  def _expand_pkg_info(pkgs, branch, repotype=None):

      ''' Return a JSON blob containing all the information we want to return

      for the provided package or packages.
@@ -156,15 +94,15 @@ 

      for pkg in pkgs:

          out = pkg.to_json()

          dbfile = '%s/mdapi-%s%s-primary.sqlite' % (

-             CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '')

+             app.config['DB_FOLDER'], branch, '-%s' % repotype if repotype else '')

  

-         session = yield from mdapilib.create_session(

+         session = mdapilib.create_session(

              'sqlite:///%s' % dbfile)

          # Fill in some extra info

  

          # Basic infos, always present regardless of the version of the repo

          for datatype in ['conflicts', 'obsoletes', 'provides', 'requires']:

-             data = yield from mdapilib.get_package_info(

+             data = mdapilib.get_package_info(

                  session, pkg.pkgKey, datatype.capitalize())

              if data:

                  out[datatype] = [item.to_json() for item in data]
@@ -174,7 +112,7 @@ 

          # New meta-data present for soft dependency management in RPM

          for datatype in [

                  'enhances', 'recommends', 'suggests', 'supplements']:

-             data = yield from mdapilib.get_package_info(

+             data = mdapilib.get_package_info(

                  session, pkg.pkgKey, datatype.capitalize())

              if data:

                  out[datatype] = [item.to_json() for item in data]
@@ -183,7 +121,7 @@ 

  

          # Add the list of packages built from the same src.rpm

          if pkg.rpm_sourcerpm:

-             copkgs = yield from mdapilib.get_co_packages(

+             copkgs = mdapilib.get_co_packages(

                  session, pkg.rpm_sourcerpm)

              out['co-packages'] = list(set([

                  cpkg.name for cpkg in copkgs
@@ -199,248 +137,129 @@ 

          return output

  

  

- @asyncio.coroutine

- @allows_jsonp

- def get_pkg(request):

-     _log.info('get_pkg %s', request)

-     branch = request.match_info.get('branch')

-     pretty = _get_pretty(request)

-     name = request.match_info.get('name')

-     pkg, repotype = yield from _get_pkg(branch, name)

- 

-     output = yield from _expand_pkg_info(pkg, branch, repotype)

+ @app.route("/<branch>/pkg/<name>")

+ def get_pkg(branch, name):

+     pkg, repotype = _get_pkg(branch, name)

+     output = _expand_pkg_info(pkg, branch, repotype)

+     return jsonify(output)

  

-     args = {}

-     if pretty:

-         args = dict(sort_keys=True, indent=4, separators=(',', ': '))

  

-     output = web.Response(

-         body=json.dumps(output, **args).encode('utf-8'),

-         content_type='application/json')

-     return output

+ @app.route("/<branch>/srcpkg/<name>")

+ def get_src_pkg(branch, name):

+     pkg, repotype = _get_pkg(branch, srcname=name)

+     output = _expand_pkg_info(pkg, branch, repotype)

  

+     return jsonify(output)

  

- @asyncio.coroutine

- @allows_jsonp

- def get_src_pkg(request):

-     _log.info('get_src_pkg %s', request)

-     branch = request.match_info.get('branch')

-     pretty = _get_pretty(request)

-     name = request.match_info.get('name')

-     pkg, repotype = yield from _get_pkg(branch, srcname=name)

  

-     output = yield from _expand_pkg_info(pkg, branch, repotype)

- 

-     args = {}

-     if pretty:

-         args = dict(sort_keys=True, indent=4, separators=(',', ': '))

- 

-     return web.Response(

-         body=json.dumps(output, **args).encode('utf-8'),

-         content_type='application/json')

- 

- 

- @asyncio.coroutine

- @allows_jsonp

- def get_pkg_files(request):

-     _log.info('get_pkg_files %s', request)

-     branch = request.match_info.get('branch')

-     name = request.match_info.get('name')

-     pretty = _get_pretty(request)

-     pkg, repotype = yield from _get_pkg(branch, name)

+ @app.route("/<branch>/files/<name>")

+ def get_pkg_files(branch, name):

+     pkg, repotype = _get_pkg(branch, name)

  

      dbfile = '%s/mdapi-%s%s-filelists.sqlite' % (

-         CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '')

+         app.config['DB_FOLDER'], branch, '-%s' % repotype if repotype else '')

      if not os.path.exists(dbfile):

-         raise web.HTTPBadRequest()

+         abort(400)

  

-     session2 = yield from mdapilib.create_session(

+     session2 = mdapilib.create_session(

          'sqlite:///%s' % dbfile)

-     filelist = yield from mdapilib.get_files(session2, pkg.pkgId)

+     filelist = mdapilib.get_files(session2, pkg.pkgId)

      session2.close()

  

      output = {

          'files': [fileinfo.to_json() for fileinfo in filelist],

          'repo': repotype if repotype else 'release',

      }

-     args = {}

-     if pretty:

-         args = dict(sort_keys=True, indent=4, separators=(',', ': '))

- 

-     return web.Response(

-         body=json.dumps(output, **args).encode('utf-8'),

-         content_type='application/json')

+     return jsonify(output)

  

  

- @asyncio.coroutine

- @allows_jsonp

- def get_pkg_changelog(request):

-     _log.info('get_pkg_changelog %s', request)

-     branch = request.match_info.get('branch')

-     name = request.match_info.get('name')

-     pretty = _get_pretty(request)

-     pkg, repotype = yield from _get_pkg(branch, name)

+ @app.route("/<branch>/changelog/<name>")

+ def get_pkg_changelog(branch, name):

+     pkg, repotype = _get_pkg(branch, name)

  

      dbfile = '%s/mdapi-%s%s-other.sqlite' % (

-         CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '')

+         app.config['DB_FOLDER'], branch, '-%s' % repotype if repotype else '')

      if not os.path.exists(dbfile):

-         raise web.HTTPBadRequest()

+         abort(400)

  

-     session2 = yield from mdapilib.create_session(

+     session2 = mdapilib.create_session(

          'sqlite:///%s' % dbfile)

-     changelogs = yield from mdapilib.get_changelog(session2, pkg.pkgId)

+     changelogs = mdapilib.get_changelog(session2, pkg.pkgId)

      session2.close()

  

      output = {

          'changelogs': [changelog.to_json() for changelog in changelogs],

          'repo': repotype if repotype else 'release',

      }

-     args = {}

-     if pretty:

-         args = dict(sort_keys=True, indent=4, separators=(',', ': '))

  

-     return web.Response(

-         body=json.dumps(output, **args).encode('utf-8'),

-         content_type='application/json')

+     return jsonify(output)

  

  

- @asyncio.coroutine

- def list_branches(request):

+ @app.route("/branches")

+ def list_branches():

      ''' Return the list of all branches currently supported by mdapi

      '''

-     _log.info('list_branches: %s', request)

-     pretty = _get_pretty(request)

      output = sorted(list(set([

          # Remove the front part `mdapi-` and the end part -<type>.sqlite

          filename.replace('mdapi-', '').rsplit('-', 2)[0].replace(

              '-updates', '')

-         for filename in os.listdir(CONFIG['DB_FOLDER'])

+         for filename in os.listdir(app.config['DB_FOLDER'])

          if filename.startswith('mdapi') and filename.endswith('.sqlite')

      ])))

  

-     args = {}

-     if pretty:

-         args = dict(sort_keys=True, indent=4, separators=(',', ': '))

+     return jsonify(output)

  

-     response = web.Response(body=json.dumps(output, **args).encode('utf-8'),

-                             content_type='application/json')

  

-     # The decorator doesn't work for this endpoint, so do it manually here

-     # I am not really sure what doesn't work but it seems this endpoint is

-     # returning an object instead of the expected generator despite it being

-     # flagged as an asyncio coroutine

-     url_arg = request.query

-     callback = url_arg.get('callback')

-     if callback and request.method == 'GET':

-         if isinstance(callback, list):

-             callback = callback[0]

-         response.mimetype = 'application/javascript'

-         response.content_type = 'application/javascript'

-         response.text = '%s(%s);' % (callback, response.text)

- 

-     return response

- 

- 

- @asyncio.coroutine

- @allows_jsonp

- def process_dep(request, action):

+ def process_dep(branch, name, action):

      ''' Return the information about the packages having the specified

      action (provides, requires, obsoletes...)

      '''

-     _log.info('process_dep %s: %s', action, request)

-     branch = request.match_info.get('branch')

-     pretty = _get_pretty(request)

-     name = request.match_info.get('name')

- 

-     try:

-         pkg, repotype = yield from _get_pkg(branch, name, action=action)

-     except:

-         raise web.HTTPBadRequest()

- 

-     output = yield from _expand_pkg_info(pkg, branch, repotype)

- 

-     args = {}

-     if pretty:

-         args = dict(sort_keys=True, indent=4, separators=(',', ': '))

- 

-     return web.Response(body=json.dumps(output, **args).encode('utf-8'),

-                         content_type='application/json')

- 

- 

- @asyncio.coroutine

- def get_provides(request):

-     return process_dep(request, 'provides')

- 

- 

- @asyncio.coroutine

- def get_requires(request):

-     return process_dep(request, 'requires')

- 

+     pkg, repotype = _get_pkg(branch, name, action=action)

+     output = _expand_pkg_info(pkg, branch, repotype)

  

- @asyncio.coroutine

- def get_obsoletes(request):

-     return process_dep(request, 'obsoletes')

+     return jsonify(output)

  

  

- @asyncio.coroutine

- def get_conflicts(request):

-     return process_dep(request, 'conflicts')

+ @app.route("/<branch>/provides/<name>")

+ def get_provides(branch, name):

+     return process_dep(branch, name, 'provides')

  

  

- @asyncio.coroutine

- def get_enhances(request):

-     return process_dep(request, 'enhances')

+ @app.route("/<branch>/requires/<name>")

+ def get_requires(branch, name):

+     return process_dep(branch, name, 'requires')

  

  

- @asyncio.coroutine

- def get_recommends(request):

-     return process_dep(request, 'recommends')

+ @app.route("/<branch>/obsoletes/<name>")

+ def get_obsoletes(branch, name):

+     return process_dep(branch, name, 'obsoletes')

  

  

- @asyncio.coroutine

- def get_suggests(request):

-     return process_dep(request, 'suggests')

+ @app.route("/<branch>/conflicts/<name>")

+ def get_conflicts(branch, name):

+     return process_dep(branch, name, 'conflicts')

  

  

- @asyncio.coroutine

- def get_supplements(request):

-     return process_dep(request, 'supplements')

+ @app.route("/<branch>/enhances/<name>")

+ def get_enhances(branch, name):

+     return process_dep(branch, name, 'enhances')

  

  

- @asyncio.coroutine

- def index(request):

-     _log.info('index %s', request)

-     return web.Response(

-         body=INDEX.encode('utf-8'),

-         content_type='text/html',

-         charset='utf-8')

+ @app.route("/<branch>/recommends/<name>")

+ def get_recommends(branch, name):

+     return process_dep(branch, name, 'recommends')

  

  

- def _set_routes(app):

-     routes = []

-     prefix = CONFIG.get('PREFIX', '')

-     if prefix:

-         routes.append(('', index))

+ @app.route("/<branch>/suggests/<name>")

+ def get_suggests(branch, name):

+     return process_dep(branch, name, 'suggests')

  

-     routes.extend([

-         ('/', index),

-         ('/branches', list_branches),

-         ('/{branch}/pkg/{name}', get_pkg),

-         ('/{branch}/srcpkg/{name}', get_src_pkg),

  

-         ('/{branch}/provides/{name}', get_provides),

-         ('/{branch}/requires/{name}', get_requires),

-         ('/{branch}/obsoletes/{name}', get_obsoletes),

-         ('/{branch}/conflicts/{name}', get_conflicts),

+ @app.route("/<branch>/supplements/<name>")

+ def get_supplements(branch, name):

+     return process_dep(branch, name, 'supplements')

  

-         ('/{branch}/enhances/{name}', get_enhances),

-         ('/{branch}/recommends/{name}', get_recommends),

-         ('/{branch}/suggests/{name}', get_suggests),

-         ('/{branch}/supplements/{name}', get_supplements),

  

-         ('/{branch}/files/{name}', get_pkg_files),

-         ('/{branch}/changelog/{name}', get_pkg_changelog),

-     ])

-     for route in routes:

-         app.router.add_route('GET', prefix + route[0], route[1])

-     return app

+ @app.route("/")

+ def index():

+     return send_from_directory(directory="", filename="index.html")

file modified
+14 -14
@@ -68,7 +68,7 @@ 

  

      /branches

  

-     <a href="$PREFIX/branches">/branches</a>

+     <a href="/branches">/branches</a>

  

  

  Note:
@@ -88,7 +88,7 @@ 

  

  So for example, for the kernel in rawhide:

  

-     <a href="$PREFIX/rawhide/pkg/kernel">/rawhide/pkg/kernel</a>

+     <a href="/rawhide/pkg/kernel">/rawhide/pkg/kernel</a>

  

  

  You can also retrieve information about a specific package on a specific
@@ -98,7 +98,7 @@ 

  

  So for example, for the python-natsort in rawhide that only exists as src.rpm:

  

-     <a href="$PREFIX/rawhide/srcpkg/python-natsort">/rawhide/srcpkg/python-natsort</a>

+     <a href="/rawhide/srcpkg/python-natsort">/rawhide/srcpkg/python-natsort</a>

  

  

  Retrieve the list of files in a package
@@ -111,7 +111,7 @@ 

  

  So for example, for the kernel-core in rawhide:

  

-     <a href="$PREFIX/rawhide/files/kernel-core">/rawhide/files/kernel-core</a>

+     <a href="/rawhide/files/kernel-core">/rawhide/files/kernel-core</a>

  

  

  Retrieve the changelog of a package
@@ -124,7 +124,7 @@ 

  

  So for example, for the kernel in rawhide:

  

-     <a href="$PREFIX/rawhide/changelog/kernel">/rawhide/changelog/kernel</a>

+     <a href="/rawhide/changelog/kernel">/rawhide/changelog/kernel</a>

  

  

  Retrieve the packages having a specific property
@@ -143,29 +143,29 @@ 

  Few examples:

  

      packages requiring R in rawhide:

-     <a href="$PREFIX/rawhide/requires/R">/rawhide/requires/R</a>

-       To see what R itself requires, check its information using: <a href="$PREFIX/rawhide/pkg/R">/rawhide/pkg/R</a>

+     <a href="/rawhide/requires/R">/rawhide/requires/R</a>

+       To see what R itself requires, check its information using: <a href="/rawhide/pkg/R">/rawhide/pkg/R</a>

  

      packages providing perl(SetupLog) in rawhide:

-     <a href="$PREFIX/rawhide/provides/perl(SetupLog)">/rawhide/provides/perl(SetupLog)</a>

+     <a href="/rawhide/provides/perl(SetupLog)">/rawhide/provides/perl(SetupLog)</a>

  

      packages obsoleting cabal2spec in rawhide:

-     <a href="$PREFIX/rawhide/obsoletes/cabal2spec">rawhide/obsoletes/cabal2spec</a>

+     <a href="/rawhide/obsoletes/cabal2spec">rawhide/obsoletes/cabal2spec</a>

  

      packages conflicting with mariadb in rawhide:

-     <a href="$PREFIX/rawhide/conflicts/mariadb">rawhide/conflicts/mariadb</a>

+     <a href="/rawhide/conflicts/mariadb">rawhide/conflicts/mariadb</a>

  

      packages enhancing httpd in rawhide:

-     <a href="$PREFIX/rawhide/enhances/httpd">rawhide/enhances/httpd</a>

+     <a href="/rawhide/enhances/httpd">rawhide/enhances/httpd</a>

  

      packages recommending flac in rawhide:

-     <a href="$PREFIX/rawhide/recommends/flac">rawhide/recommends/flac</a>

+     <a href="/rawhide/recommends/flac">rawhide/recommends/flac</a>

  

      packages suggesting R-tools in rawhide:

-     <a href="$PREFIX/rawhide/suggests/R-tools">rawhide/suggests/R-tools</a>

+     <a href="/rawhide/suggests/R-tools">rawhide/suggests/R-tools</a>

  

      packages supplementing `(hunspell and langpacks-fr)` in rawhide:

-     <a href="$PREFIX/rawhide/supplements/(hunspell and langpacks-fr)">rawhide/supplements/(hunspell and langpacks-fr)</a>

+     <a href="/rawhide/supplements/(hunspell and langpacks-fr)">rawhide/supplements/(hunspell and langpacks-fr)</a>

  

  

  

file modified
+8 -33
@@ -23,10 +23,8 @@ 

  MDAPI internal API to interact with the database.

  '''

  

- import contextlib

  import time

  

- import asyncio

  import sqlalchemy as sa

  

  from sqlalchemy.orm import sessionmaker
@@ -41,7 +39,6 @@ 

  RETRY_ATTEMPT = 3

  

  

- @asyncio.coroutine

  def create_session(db_url, debug=False, pool_recycle=3600):

      """ Create the Session object to use to query the database.

  
@@ -60,22 +57,6 @@ 

      return scopedsession

  

  

- @contextlib.contextmanager

- def session_manager(db_url, debug=False, pool_recycle=3600):

-     """ A handy context manager for our sessions. """

-     session = yield from create_session(

-         db_url, debug=debug, pool_recycle=pool_recycle)

-     try:

-         yield session

-         session.commit()

-     except:

-         session.rollback()

-         raise

-     finally:

-         session.close()

- 

- 

- @asyncio.coroutine

  def get_package(session, pkg_name):

      ''' Return information about a package, if we can find it.

      '''
@@ -98,17 +79,16 @@ 

              raise

          else:

              time.sleep(0.1)

-             output = yield from get_package(session, pkg_name)

+             output = get_package(session, pkg_name)

  

      return output

  

  

- @asyncio.coroutine

  def get_package_by_src(session, pkg_name):

      ''' Return information about a package, if we can find it.

      '''

      # First try if there is a package matching exactly the provided name

-     simple_match = yield from get_package(session, pkg_name)

+     simple_match = get_package(session, pkg_name)

      if simple_match and simple_match.basename == pkg_name:

          return simple_match

  
@@ -133,10 +113,9 @@ 

              raise

          else:

              time.sleep(0.1)

-             yield from get_package_by_src(session, pkg_name)

+             get_package_by_src(session, pkg_name)

  

  

- @asyncio.coroutine

  def get_package_by(session, tablename, key, cnt=None):

      ''' Return information the package providing the provides, if we can find it.

      '''
@@ -163,13 +142,12 @@ 

              raise

          else:

              time.sleep(0.1)

-             output = yield from get_package_by(

+             output = get_package_by(

                  session, tablename, key, cnt=cnt)

  

      return output

  

  

- @asyncio.coroutine

  def get_package_info(session, pkgKey, tablename):

      ''' Return the information contained in the specified table for the

      given package.
@@ -192,12 +170,11 @@ 

              raise

          else:

              time.sleep(0.1)

-             output = yield from get_package_info(session, pkgKey, tablename)

+             output = get_package_info(session, pkgKey, tablename)

  

      return output

  

  

- @asyncio.coroutine

  def get_co_packages(session, srcpkg_name):

      ''' Return the name of all the packages coming from the same

      source-package.
@@ -217,12 +194,11 @@ 

              raise

          else:

              time.sleep(0.1)

-             output = yield from get_co_packages(session, srcpkg_name)

+             output = get_co_packages(session, srcpkg_name)

  

      return output

  

  

- @asyncio.coroutine

  def get_files(session, pkg_id):

      ''' Return the list of all the files in a package given its key.

      '''
@@ -244,12 +220,11 @@ 

              raise

          else:

              time.sleep(0.1)

-             output = yield from get_files(session, pkg_id)

+             output = get_files(session, pkg_id)

  

      return output

  

  

- @asyncio.coroutine

  def get_changelog(session, pkg_id):

      ''' Return the list of all the changelog in a package given its key.

      '''
@@ -271,6 +246,6 @@ 

              raise

          else:

              time.sleep(0.1)

-             output = yield from get_changelog(session, pkg_id)

+             output = get_changelog(session, pkg_id)

  

      return output

file removed
-20
@@ -1,20 +0,0 @@ 

- import logging

- import logging.config

- 

- from aiohttp import web

- 

- from mdapi import CONFIG, _set_routes

- 

- 

- def main():

- 

-     logging.basicConfig()

-     logging.config.dictConfig(CONFIG.get("LOGGING") or {"version": 1})

- 

-     app = web.Application()

-     app = _set_routes(app)

- 

-     host = CONFIG.get("HOST", "127.0.0.1")

-     port = CONFIG.get("PORT", 8080)

- 

-     web.run_app(app, host=host, port=port)

file modified
+2 -4
@@ -1,7 +1,5 @@ 

- aiohttp >= 3.5.4

+ flask

  fedora_messaging

- # this is a requirement of aiohttp but better safe than sorry

  requests

  sqlalchemy

- werkzeug

- flufl.lock

+ gunicorn

file modified
+42 -62
@@ -27,24 +27,18 @@ 

  '''

  import json

  import os

- import shutil

  import subprocess

  import sys

- import tempfile

  

- import mock

+ 

  import pytest

- from aiohttp import web

- from sqlalchemy.exc import SQLAlchemyError

+ from mdapi import app

  

  sys.path.insert(0, os.path.join(os.path.dirname(

      os.path.abspath(__file__)), '..'))

  

- import mdapi

- 

  HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)))

  

- TMPDIR = None

  

  @pytest.fixture(scope="module")

  def set_env(request):
@@ -52,43 +46,33 @@ 

      Collects the sqlite database from the mirror to have some data to test

      against.

      """

-     global TMPDIR

-     TMPDIR = tempfile.mkdtemp(prefix="mdapi-test-")

-     print("Creating %s" % TMPDIR)

-     configfile = os.path.join(TMPDIR, "config")

+ 

+     tmpdir = "/tmp"

+     configfile = os.path.join(tmpdir, "config")

  

      with open(configfile, "w") as stream:

-         stream.write("DB_FOLDER = '%s'\n" % TMPDIR)

+         stream.write("DB_FOLDER = '%s'\n" % tmpdir)

  

      print("Downloading the databases...")

      subprocess.check_output(

          ["../mdapi-get_repo_md", configfile],

          cwd=HERE,

      )

-     assert len(os.listdir(TMPDIR)) > 2

- 

-     def clean_up():

-         print("\nRemoving %s" % TMPDIR)

-         shutil.rmtree(TMPDIR)

-     request.addfinalizer(clean_up)

+     assert len(os.listdir(tmpdir)) > 2

  

  

  @pytest.fixture

- def tmpdir():

-     return TMPDIR

- 

+ def client(set_env, tmpdir):

+     app.config['TESTING'] = True

+     app.config['DB_FOLDER'] = "/tmp"

  

- @pytest.fixture

- def cli(set_env, tmpdir, loop, aiohttp_client):

-     mdapi.CONFIG['DB_FOLDER'] = tmpdir

-     app = web.Application()

-     app = mdapi._set_routes(app)

-     return loop.run_until_complete(aiohttp_client(app))

+     with app.test_client() as client:

+         yield client

  

  

- async def test_view_index_page(cli):

-     resp = await cli.get('/')

-     assert resp.status == 200

+ def test_view_index_page(client):

+     resp = client.get('/')

+     assert resp.status_code == 200

      header = r"""

                 _             _

                | |           (_)
@@ -99,51 +83,47 @@ 

                        | |

                        |_|

  """

-     output = await resp.text()

-     assert header in output

+     assert header in resp.get_data(as_text=True)

  

  

- async def test_view_branches(cli):

-     resp = await cli.get('/branches')

-     assert resp.status == 200

-     output = await resp.text()

+ def test_view_branches(client):

+     resp = client.get('/branches')

+     assert resp.status_code == 200

+     output = resp.get_data(as_text=True)

      assert 'src_rawhide' in output

      assert 'rawhide' in output

  

  

- async def test_view_pkg_rawhide(cli):

-     resp = await cli.get('/rawhide/pkg/kernel')

-     assert resp.status == 200

-     json.loads(await resp.text())

+ def test_view_pkg_rawhide(client):

+     resp = client.get('/rawhide/pkg/kernel')

+     assert resp.status_code == 200

  

  

- async def test_view_pkg_rawhide_invalid(cli):

-     resp = await cli.get('/rawhide/pkg/invalidpackagename')

-     assert resp.status == 404

-     assert '404: Not Found' == await resp.text()

+ def test_view_pkg_rawhide_invalid(client):

+     resp = client.get('/rawhide/pkg/invalidpackagename')

+     assert resp.status_code == 404

  

  

- async def test_view_srcpkg_rawhide(cli):

-     resp = await cli.get('/rawhide/srcpkg/python-natsort')

-     assert resp.status == 200

-     json.loads(await resp.text())

+ def test_view_srcpkg_rawhide(client):

+     resp = client.get('/rawhide/srcpkg/python-natsort')

+     assert resp.status_code == 200

+     json.loads(resp.get_data(as_text=True))

  

  

- async def test_view_file_list_rawhide(cli):

-     resp = await cli.get('/rawhide/files/kernel-core')

-     assert resp.status == 200

-     json.loads(await resp.text())

+ def test_view_file_list_rawhide(client):

+     resp = client.get('/rawhide/files/kernel-core')

+     assert resp.status_code == 200

+     json.loads(resp.get_data(as_text=True))

  

  

- async def test_view_changelog_rawhide(cli):

-     resp = await cli.get('/rawhide/changelog/kernel')

-     assert resp.status == 200

-     json.loads(await resp.text())

+ def test_view_changelog_rawhide(client):

+     resp = client.get('/rawhide/changelog/kernel')

+     assert resp.status_code == 200

+     json.loads(resp.get_data(as_text=True))

  

  

  @pytest.mark.parametrize("action, package, status_code", [

      ("requires", "R", 200),

-     ("provides", "perl(SetupLog)", 200),

      ("provides", "R", 200),

      ("obsoletes", "cabal2spec", 200),

      ("conflicts", "mariadb", 200),
@@ -152,8 +132,8 @@ 

      ("suggests", "httpd", 200),

      ("supplements", "(hunspell and langpacks-fr)", 200),

  ])

- async def test_view_property_koji(cli, action, package, status_code):

-     resp = await cli.get('/koji/%s/%s' % (action, package))

-     assert resp.status == status_code

+ def test_view_property_koji(client, action, package, status_code):

+     resp = client.get('/koji/%s/%s' % (action, package))

+     assert resp.status_code == status_code

      if status_code == 200:

-         json.loads(await resp.text())

+         json.loads(resp.get_data(as_text=True))

file modified
+34 -45
@@ -26,34 +26,29 @@ 

  

  '''

  import os

- import shutil

- import subprocess

  import sys

- import tempfile

  

  import pytest

- from aiohttp import web

+ from mdapi import app

  

  sys.path.insert(0, os.path.join(os.path.dirname(

      os.path.abspath(__file__)), '..'))

  

- import mdapi

- import mdapi.lib

- 

  HERE = os.path.join(os.path.dirname(os.path.abspath(__file__)))

  

  

  @pytest.fixture

- def cli(loop, aiohttp_client):

-     mdapi.CONFIG['DB_FOLDER'] = '.'

-     app = web.Application()

-     app = mdapi._set_routes(app)

-     return loop.run_until_complete(aiohttp_client(app))

+ def client():

+     app.config['TESTING'] = True

+     app.config['DB_FOLDER'] = ""

+ 

+     with app.test_client() as client:

+         yield client

  

  

- async def test_view_index_page(cli):

-     resp = await cli.get('/')

-     assert resp.status == 200

+ def test_view_index_page(client):

+     resp = client.get('/')

+     assert resp.status_code == 200

      header = r"""

                 _             _

                | |           (_)
@@ -64,51 +59,45 @@ 

                        | |

                        |_|

  """

-     output = await resp.text()

-     assert header in output

+     assert header in resp.get_data(as_text=True)

  

  

- async def test_view_branches(cli):

-     resp = await cli.get('/branches')

-     assert resp.status == 200

-     assert '[]' == await resp.text()

+ def test_view_branches(client):

+     app.config['DB_FOLDER'] = "/"

+     resp = client.get('/branches')

+     assert resp.status_code == 200

+     assert '[]\n' == resp.get_data(as_text=True)

  

  

- async def test_view_pkg_rawhide(cli):

-     resp = await cli.get('/rawhide/pkg/kernel')

-     assert resp.status == 400

-     assert '400: Bad Request' == await resp.text()

+ def test_view_pkg_rawhide(client):

+     resp = client.get('/rawhide/pkg/kernel')

+     assert resp.status_code == 400

  

  

- async def test_view_pkg_rawhide_invalid(cli):

-     resp = await cli.get('/rawhide/pkg/invalidpackagename')

-     assert resp.status == 400

-     assert '400: Bad Request' == await resp.text()

+ def test_view_pkg_rawhide_invalid(client):

+     resp = client.get('/rawhide/pkg/invalidpackagename')

+     assert resp.status_code == 400

  

  

- async def test_view_srcpkg_rawhide(cli):

-     resp = await cli.get('/rawhide/srcpkg/python-natsort')

-     assert resp.status == 400

-     assert '400: Bad Request' == await resp.text()

+ def test_view_srcpkg_rawhide(client):

+     resp = client.get('/rawhide/srcpkg/python-natsort')

+     assert resp.status_code == 400

  

  

- async def test_view_file_list_rawhide(cli):

-     resp = await cli.get('/rawhide/files/kernel-core')

-     assert resp.status == 400

-     assert '400: Bad Request' == await resp.text()

+ def test_view_file_list_rawhide(client):

+     resp = client.get('/rawhide/files/kernel-core')

+     assert resp.status_code == 400

  

  

- async def test_view_changelog_rawhide(cli):

-     resp = await cli.get('/rawhide/changelog/kernel')

-     assert resp.status == 400

-     assert '400: Bad Request' == await resp.text()

+ def test_view_changelog_rawhide(client):

+     resp = client.get('/rawhide/changelog/kernel')

+     assert resp.status_code == 400

  

  

  @pytest.mark.parametrize("action", [

      "requires", "provides", "obsoletes", "conflicts",

      "enhances", "recommends", "suggests", "supplements",

  ])

- async def test_view_property_koji(cli, action):

-     resp = await cli.get('/koji/%s/R' % action)

-     assert resp.status == 400

-     assert '400: Bad Request' == await resp.text()

+ def test_view_property_koji(client, action):

+     resp = client.get('/koji/%s/R' % action)

+     assert resp.status_code == 400

This commit moves mdapi from the aiohttp framework to flask. This
is done since there is little interest in using aiohttp because all
the call to sqlite are io blocking. There is no asyncio driver for sqlalchemy
and sqlite.
Also performance with Flask and 4 gunicorn workers are better than the
performances achieved with aiohttp.

Signed-off-by: Clement Verna cverna@tutanota.com

rebased onto 99321b95f0f82088a82faf2e44d122a6a80f2f9b

5 years ago

rebased onto 8036b1a4fc651a410f3f0f267a1131b918fcbacc

5 years ago

rebased onto 670020c

5 years ago

Pull-Request has been closed by cverna

5 years ago

Hey @cverna is there a specific reason you closed this PR?
I thought you said it made mdapi simpler, faster and more reliable?

If that's the case, I think I'd like to take the time to review it :)

Hey @cverna is there a specific reason you closed this PR?
I thought you said it made mdapi simpler, faster and more reliable?
If that's the case, I think I'd like to take the time to review it :)

Yes it is still quite slow which makes the pod crash in OpenShift (when indexing packages the health probe are failing which cause the pod to restart). The main reason I looked at flask was because I did not find any asyncio library for sqlite but I came across one (https://github.com/encode/databases) so I think that might be a simpler change to use this for all the db calls and not have blocking call to sqlite.

I also need to look at the packages script to see if it could be a bit less aggressive towards mdapi.