From 275f79e6337350a2d8d46b8433149037af487105 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Nov 15 2019 09:24:26 +0000 Subject: [PATCH 1/2] Replace old string formating by f-strings Signed-off-by: Clement Verna --- diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md index 0fb31f2..ee4590a 100755 --- a/mdapi-get_repo_md +++ b/mdapi-get_repo_md @@ -153,15 +153,14 @@ def list_branches(status='Active'): ''' Return the list of Fedora branches corresponding to the given status. ''' - url = PKGDB2_URL + 'api/collections?clt_status=%s' % status + url = PKGDB2_URL + f'api/collections?clt_status={status}' response = requests.get(url, verify=PKGDB2_VERIFY) data = response.json() return data['collections'] def download_db(name, repomd_url, archive): - print('%s Downloading file: %s to %s' % ( - name.ljust(padding), repomd_url, archive)) + print(f'{name.ljust(padding)} Downloading file: {repomd_url} to {archive}') response = requests.get(repomd_url, verify=DL_VERIFY) with open(archive, 'wb') as stream: stream.write(response.content) @@ -169,7 +168,7 @@ def download_db(name, repomd_url, archive): def decompress_db(name, archive, location): ''' Decompress the given XZ archive at the specified location. ''' - print('%s Extracting %s to %s' % (name.ljust(padding), archive, location)) + print(f'{name.ljust(padding)} Extracting {archive} to {location}') if archive.endswith('.xz'): import lzma with contextlib.closing(lzma.LZMAFile(archive)) as stream_xz: @@ -196,7 +195,7 @@ def decompress_db(name, archive, location): def compare_dbs(name, db1, db2, cache1, cache2): - print('%s Comparing %s and %s' % (name.ljust(padding), db1, db2)) + print(f'{name.ljust(padding)} Comparing {db1} and {db2}') def get_table_names(uri): with mdapilib.session_manager('sqlite:///' + uri) as session: @@ -223,10 +222,8 @@ def compare_dbs(name, db1, db2, cache1, cache2): row[0] = cache[row[0]] yield tuple(row) else: - print("%s ! %r does not appear in the " - "%r cache for %r. Dropping " - "from comparison." % ( - name.ljust(padding), row[0], table, uri)) + print(f"{name.ljust(padding)} ! {row[0]!r} does not appear in the " + f"{table!r} cache for {uri}. Dropping from comparison.") else: yield tuple(row) @@ -248,7 +245,7 @@ def compare_dbs(name, db1, db2, cache1, cache2): # We have never downloaded this before... # so we have nothing to compare it against. Just return and say there # are "no differences". - print('%s Empty! %s Cannot compare.' % (name.ljust(padding), db2)) + print(f'{name.ljust(padding)} Empty! {db2} Cannot compare.') return set() assert len(tables1) == len(tables2), "Cannot compare disparate dbs." @@ -293,11 +290,11 @@ def compare_dbs(name, db1, db2, cache1, cache2): def publish_changes(name, packages, repomd_url): - print('%s Publishing differences to fedora messaging:' % (name.ljust(padding))) + print(f'{name.ljust(padding)} Publishing differences to fedora messaging:') change = bool(packages) if not change: - print('%s No real changes. Skipping fedora messaging.' % (name.ljust(padding))) + print(f'{name.ljust(padding)} No real changes. Skipping fedora messaging.') return # Just publish the suffix of the URL. The prefix is dl.fedoraproject.org @@ -306,7 +303,7 @@ def publish_changes(name, packages, repomd_url): # download.fedoraproject.org.. so, just obscure *exactly* which repo we're # talking about. url = '/'.join(repomd_url.split('/')[4:]) - print("%s url %s" % (name.ljust(padding), url)) + print(f"{name.ljust(padding)} url {url}") try: msg = Message( @@ -327,7 +324,7 @@ def publish_changes(name, packages, repomd_url): def install_db(name, src, dest): - print('%s Installing %s to %s.' % (name.ljust(padding), src, dest)) + print(f'{name.ljust(padding)} Installing {src} to {dest}.') shutil.move(src, dest) @@ -365,8 +362,7 @@ def process_repo(tupl): repomd_url = url + '/repomd.xml' response = requests.get(repomd_url, verify=DL_VERIFY) if not bool(response): - print('%s !! Failed to get %r %r' % ( - name.ljust(padding), repomd_url, response)) + print(f'{name.ljust(padding)} !! Failed to get {repomd_url!r} {response!r}') return # Parse the xml doc and get a list of locations and their shasum. @@ -393,7 +389,7 @@ def process_repo(tupl): cache1, cache2 = {}, {} if not files: - print('No sqlite database could be found in %s' % url) + print(f'No sqlite database could be found in {url}') for filename, shasum, shatype in files: repomd_url = url + '/' + filename @@ -401,16 +397,16 @@ def process_repo(tupl): # First, determine if the file has changed by comparing hash db = None if 'primary.sqlite' in filename: - db = 'mdapi-%s-primary.sqlite' % name + db = f'mdapi-{name}-primary.sqlite' elif 'filelists.sqlite' in filename: - db = 'mdapi-%s-filelists.sqlite' % name + db = f'mdapi-{name}-filelists.sqlite' elif 'other.sqlite' in filename: - db = 'mdapi-%s-other.sqlite' % name + db = f'mdapi-{name}-other.sqlite' # Have we downloaded this before? Did it change? destfile = os.path.join(destfolder, db) if not needs_update(destfile, shasum, shatype): - print('%s No change of %s' % (name.ljust(padding), repomd_url)) + print(f'{name.ljust(padding)} No change of {repomd_url}') continue # If it has changed, then download it and move it into place. @@ -469,15 +465,14 @@ def main(): version = release['version'] if version == 'devel': version = 'rawhide' - url = '%s/pub/fedora/linux/' \ - 'development/%s/Everything/x86_64/os/repodata' % (DL_SERVER, version) + url = f'{DL_SERVER}/pub/fedora/linux/development/{version}/Everything/x86_64/os/repodata' print(release['koji_name'], version, release['status'], url) repositories.append( (url, release['koji_name']) ) url = url.replace('/x86_64/os/', '/source/tree/') - repositories.append((url, 'src_%s' % release['koji_name'])) + repositories.append((url, f'src_{release["koji_name"]}')) urls = { 'Fedora': @@ -513,7 +508,7 @@ def main(): repositories.append((rurl, name)) rurl = rurl.replace('/x86_64/os', '/source/tree') - repositories.append((rurl, 'src_%s' % name)) + repositories.append((rurl, f'src_{name}')) # In parallel #p = multiprocessing.Pool(10) diff --git a/mdapi/__init__.py b/mdapi/__init__.py index d0525f5..071f881 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -77,7 +77,7 @@ def allows_jsonp(function): callback = callback[0] response.mimetype = 'application/javascript' response.content_type = 'application/javascript' - response.text = '%s(%s);' % (callback, response.text) + response.text = f'{callback}({response.text});' return response @@ -97,11 +97,9 @@ def _get_pkg(branch, name=None, action=None, srcname=None): for repotype in ['updates-testing', 'updates', 'testing', None]: if repotype: - dbfile = '%s/mdapi-%s-%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch, repotype) + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}-{repotype}-primary.sqlite' else: - dbfile = '%s/mdapi-%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch) + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}-primary.sqlite' if not os.path.exists(dbfile): wrongdb = True @@ -109,8 +107,7 @@ def _get_pkg(branch, name=None, action=None, srcname=None): wrongdb = False - session = yield from mdapilib.create_session( - 'sqlite:///%s' % dbfile) + session = yield from mdapilib.create_session(f'sqlite:///{dbfile}') if name: if action: pkg = yield from mdapilib.get_package_by( @@ -155,11 +152,9 @@ def _expand_pkg_info(pkgs, branch, repotype=None): output = [] for pkg in pkgs: out = pkg.to_json() - dbfile = '%s/mdapi-%s%s-primary.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}-primary.sqlite' - session = yield from mdapilib.create_session( - 'sqlite:///%s' % dbfile) + session = yield from mdapilib.create_session(f'sqlite:///{dbfile}') # Fill in some extra info # Basic infos, always present regardless of the version of the repo @@ -202,7 +197,7 @@ def _expand_pkg_info(pkgs, branch, repotype=None): @asyncio.coroutine @allows_jsonp def get_pkg(request): - _log.info('get_pkg %s', request) + _log.info(f'get_pkg {request}') branch = request.match_info.get('branch') pretty = _get_pretty(request) name = request.match_info.get('name') @@ -223,7 +218,7 @@ def get_pkg(request): @asyncio.coroutine @allows_jsonp def get_src_pkg(request): - _log.info('get_src_pkg %s', request) + _log.info(f'get_src_pkg {request}') branch = request.match_info.get('branch') pretty = _get_pretty(request) name = request.match_info.get('name') @@ -243,19 +238,17 @@ def get_src_pkg(request): @asyncio.coroutine @allows_jsonp def get_pkg_files(request): - _log.info('get_pkg_files %s', request) + _log.info(f'get_pkg_files {request}') branch = request.match_info.get('branch') name = request.match_info.get('name') pretty = _get_pretty(request) pkg, repotype = yield from _get_pkg(branch, name) - dbfile = '%s/mdapi-%s%s-filelists.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}-filelists.sqlite' if not os.path.exists(dbfile): raise web.HTTPBadRequest() - session2 = yield from mdapilib.create_session( - 'sqlite:///%s' % dbfile) + session2 = yield from mdapilib.create_session(f'sqlite:///{dbfile}') filelist = yield from mdapilib.get_files(session2, pkg.pkgId) session2.close() @@ -275,19 +268,17 @@ def get_pkg_files(request): @asyncio.coroutine @allows_jsonp def get_pkg_changelog(request): - _log.info('get_pkg_changelog %s', request) + _log.info(f'get_pkg_changelog {request}') branch = request.match_info.get('branch') name = request.match_info.get('name') pretty = _get_pretty(request) pkg, repotype = yield from _get_pkg(branch, name) - dbfile = '%s/mdapi-%s%s-other.sqlite' % ( - CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}-other.sqlite' if not os.path.exists(dbfile): raise web.HTTPBadRequest() - session2 = yield from mdapilib.create_session( - 'sqlite:///%s' % dbfile) + session2 = yield from mdapilib.create_session(f'sqlite:///{dbfile}') changelogs = yield from mdapilib.get_changelog(session2, pkg.pkgId) session2.close() @@ -308,7 +299,7 @@ def get_pkg_changelog(request): def list_branches(request): ''' Return the list of all branches currently supported by mdapi ''' - _log.info('list_branches: %s', request) + _log.info(f'list_branches: {request}') pretty = _get_pretty(request) output = sorted(list(set([ # Remove the front part `mdapi-` and the end part -.sqlite @@ -336,7 +327,7 @@ def list_branches(request): callback = callback[0] response.mimetype = 'application/javascript' response.content_type = 'application/javascript' - response.text = '%s(%s);' % (callback, response.text) + response.text = f'{callback}({response.text});' return response @@ -347,7 +338,7 @@ def process_dep(request, action): ''' Return the information about the packages having the specified action (provides, requires, obsoletes...) ''' - _log.info('process_dep %s: %s', action, request) + _log.info(f'process_dep {action}: {request}') branch = request.match_info.get('branch') pretty = _get_pretty(request) name = request.match_info.get('name') @@ -409,7 +400,7 @@ def get_supplements(request): @asyncio.coroutine def index(request): - _log.info('index %s', request) + _log.info(f'index {request}') return web.Response( body=INDEX.encode('utf-8'), content_type='text/html', diff --git a/tests/test_mdapi_data.py b/tests/test_mdapi_data.py index 0922061..a815cc4 100644 --- a/tests/test_mdapi_data.py +++ b/tests/test_mdapi_data.py @@ -54,11 +54,11 @@ def set_env(request): """ global TMPDIR TMPDIR = tempfile.mkdtemp(prefix="mdapi-test-") - print("Creating %s" % TMPDIR) + print(f"Creating {TMPDIR}") configfile = os.path.join(TMPDIR, "config") with open(configfile, "w") as stream: - stream.write("DB_FOLDER = '%s'\n" % TMPDIR) + stream.write(f"DB_FOLDER = '{TMPDIR}'\n") print("Downloading the databases...") subprocess.check_output( @@ -68,7 +68,7 @@ def set_env(request): assert len(os.listdir(TMPDIR)) > 2 def clean_up(): - print("\nRemoving %s" % TMPDIR) + print(f"\nRemoving {TMPDIR}") shutil.rmtree(TMPDIR) request.addfinalizer(clean_up) @@ -153,7 +153,7 @@ async def test_view_changelog_rawhide(cli): ("supplements", "(hunspell and langpacks-fr)", 200), ]) async def test_view_property_koji(cli, action, package, status_code): - resp = await cli.get('/koji/%s/%s' % (action, package)) + resp = await cli.get(f'/koji/{action}/{package}') assert resp.status == status_code if status_code == 200: json.loads(await resp.text()) diff --git a/tests/test_mdapi_empty.py b/tests/test_mdapi_empty.py index 484cb8b..92f3160 100644 --- a/tests/test_mdapi_empty.py +++ b/tests/test_mdapi_empty.py @@ -109,6 +109,6 @@ async def test_view_changelog_rawhide(cli): "enhances", "recommends", "suggests", "supplements", ]) async def test_view_property_koji(cli, action): - resp = await cli.get('/koji/%s/R' % action) + resp = await cli.get(f'/koji/{action}/R') assert resp.status == 400 assert '400: Bad Request' == await resp.text() From defa131d534999c37c7de82b3061f7ca6eacbe37 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Nov 22 2019 13:36:07 +0000 Subject: [PATCH 2/2] Remove blocking call to sqlite by using aiosqlite module. This commit removes the dependency to sqlalchemy to manage the sqlite connection and queries. The reason behind is that sqlalchemy is blocking the event loop in an asyncio context which made each request needed to have access to the sqlite databases blocking. The commit replaces sqlalchemy by aiosqlite a async sqlite wrapper around the sqlite3 module from the standard library. In order to make it easier to understand the code base, all the queries used and the classes used to store the queries results were moved to the mdapi/db.py module. The views are now stored in the mdapi.views.py and added to the application router in mdapi/server.py. The logic which forms the response to the requests based on the data gathered in the databases is present in mdapi/__init__.py. It might make sense to move this to the db.py module in a later commit. Before this commit mdapi answered an average of 28.5 requests per seconds for 100 concurent requests. After this commit mdapi answers an average of 97 requests per seconds for 100 concurent requests. Signed-off-by: Clement Verna --- diff --git a/.gitignore b/.gitignore index bc39beb..b1b11f2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ dist/ alembic.ini .tox/ .pytest_cache/ +venv/ diff --git a/Dockerfile b/Dockerfile index 8fce400..83c8d2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,16 @@ # This Dockerfile is used to build the mdapi service on Openshift # mdapi.cfg configuration is managed by Openshift as a configmap -FROM registry.fedoraproject.org/fedora:latest +FROM fedora:31 LABEL maintainer "Clément Verna " EXPOSE 8080 -RUN dnf -y install python3-aiohttp python3-werkzeug python3-requests python3-sqlalchemy python3-fedora-messaging +RUN dnf -y install python3-aiohttp python3-werkzeug python3-requests python3-fedora-messaging python3-uvloop python3-pip python3-gunicorn\ + && dnf clean all \ + && pip3 install aiosqlite -USER 1001 ENV MDAPI_CONFIG=/etc/mdapi/mdapi.cfg COPY . /code -ENTRYPOINT ["/code/mdapi-run"] +WORKDIR /code +ENTRYPOINT ["gunicorn", "mdapi.server:init_app", "--bind", "0.0.0.0:8080", "--worker-class", "aiohttp.GunicornUVLoopWebWorker", "-w", "2"] diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md index ee4590a..67a9568 100755 --- a/mdapi-get_repo_md +++ b/mdapi-get_repo_md @@ -37,7 +37,6 @@ sqlite database are retrieved from the master Fedora mirror: import argparse import contextlib import itertools -import multiprocessing import os import shutil import tempfile @@ -45,15 +44,14 @@ import time import hashlib import xml.etree.ElementTree as ET import sys +import sqlite3 import requests -from sqlalchemy import text from fedora_messaging.api import Message, publish from fedora_messaging.exceptions import PublishReturned, ConnectionException -import mdapi.lib as mdapilib KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos/' PKGDB2_URL = 'https://admin.fedoraproject.org/pkgdb/' @@ -198,11 +196,12 @@ def compare_dbs(name, db1, db2, cache1, cache2): print(f'{name.ljust(padding)} Comparing {db1} and {db2}') def get_table_names(uri): - with mdapilib.session_manager('sqlite:///' + uri) as session: - for name in session.connection().engine.table_names(): - if name == 'db_info': - continue - yield name + conn = sqlite3.connect(uri) + for name in conn.execute("SELECT name FROM sqlite_master WHERE type='table'"): + if name[0] == 'db_info': + continue + yield name[0] + conn.close() def row_to_package(row): if '/' in row[0]: @@ -212,28 +211,27 @@ def compare_dbs(name, db1, db2, cache1, cache2): return name.split('(')[0] def get_all_rows(uri, table, cache): - query = text(queries.get(table, default_query).format(table=table)) - with mdapilib.session_manager('sqlite:///' + uri) as session: - engine = session.connection().engine - for i, row in enumerate(engine.execute(query)): - if table in cache_dependant_tables: - row = list(row) # lists support item assignment - if row[0] in cache: - row[0] = cache[row[0]] - yield tuple(row) - else: - print(f"{name.ljust(padding)} ! {row[0]!r} does not appear in the " - f"{table!r} cache for {uri}. Dropping from comparison.") - else: + conn = sqlite3.connect(uri) + query = queries.get(table, default_query).format(table=table) + for i, row in enumerate(conn.execute(query)): + if table in cache_dependant_tables: + row = list(row) # lists support item assignment + if row[0] in cache: + row[0] = cache[row[0]] yield tuple(row) - + else: + print(f"{name.ljust(padding)} ! {row[0]!r} does not appear in the " + f"{table!r} cache for {uri}. Dropping from comparison.") + else: + yield tuple(row) + conn.close() def build_cache(uri, cache): - query = text(packages_cache_builder.format(table=table)) - with mdapilib.session_manager('sqlite:///' + uri) as session: - engine = session.connection().engine - for pkgId, pkgname in engine.execute(query): - cache[pkgId] = pkgname + conn = sqlite3.connect(uri) + query = queries.get(table, default_query).format(table=table) + for pkgId, pkgname in conn.execute(query): + cache[pkgId] = pkgname + conn.close() tables1 = list(get_table_names(db1)) tables2 = list(get_table_names(db2)) @@ -267,6 +265,7 @@ def compare_dbs(name, db1, db2, cache1, cache2): # Same goes for the 'packages' table in the 'other' db. ('other', 'packages'), ] + def should_compare(table): for test, target in ignored_db_tables: if test in db1 and table == target: @@ -381,7 +380,7 @@ def process_repo(tupl): files = ((f, s, t) for f, s, t in files if '.sqlite' in f) # We need to ensure the primary db comes first so we can build a pkey cache - primary_first = lambda item: not 'primary' in item[0] + primary_first = lambda item: 'primary' not in item[0] files = sorted(files, key=primary_first) # Primary-key caches built from the primary dbs so we can make sense @@ -504,18 +503,18 @@ def main(): url = url.replace('/x86_64/', '/Everything/x86_64/') else: name = epel_repos[idx] % release['koji_name'] - rurl = url % (DL_SERVER, version) + rurl = url % (DL_SERVER, version) repositories.append((rurl, name)) rurl = rurl.replace('/x86_64/os', '/source/tree') repositories.append((rurl, f'src_{name}')) # In parallel - #p = multiprocessing.Pool(10) - #p.map(process_repo, itertools.product( + # p = multiprocessing.Pool(10) + # p.map(process_repo, itertools.product( # [CONFIG.get('DB_FOLDER', '/var/tmp')], # repositories) - #) + # ) # In serial sleep_for = CONFIG.get('CRON_SLEEP', 30) diff --git a/mdapi-run b/mdapi-run index d6907f3..6876aaf 100755 --- a/mdapi-run +++ b/mdapi-run @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 +#!/bin/bash -from mdapi.server import main - -main() +gunicorn mdapi.server:init_app --bind 0.0.0.0:8080 --worker-class aiohttp.GunicornUVLoopWebWorker diff --git a/mdapi/__init__.py b/mdapi/__init__.py index 071f881..ba0c131 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright © 2015 Red Hat, Inc. +# Copyright © 2015-2019 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions @@ -22,16 +22,27 @@ ''' Top level of the mdapi aiohttp application. ''' -import functools -import json import logging import os -import asyncio +import aiosqlite import werkzeug + from aiohttp import web -import mdapi.lib as mdapilib +from mdapi.db import ( + GET_PACKAGE, + GET_PACKAGE_INFO, + GET_CO_PACKAGE, + GET_PACKAGE_BY_SRC, + GET_PACKAGE_BY, + GET_FILES, + GET_CHANGELOGS, + Packages, + Dependencies, + FileList, + ChangeLog +) CONFIG = dict() @@ -46,46 +57,11 @@ if 'MDAPI_CONFIG' in os.environ and os.path.exists(os.environ['MDAPI_CONFIG']): exec(compile( config_file.read(), os.environ['MDAPI_CONFIG'], 'exec'), CONFIG) -indexfile = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'index.html') -INDEX = '' -with open(indexfile) as stream: - INDEX = stream.read() - INDEX = INDEX.replace('$PREFIX', CONFIG.get('PREFIX', '')) - _log = logging.getLogger(__name__) -def allows_jsonp(function): - ''' Add support for JSONP queries to the endpoint decorated. ''' - - @functools.wraps(function) - def wrapper(request, *args, **kwargs): - ''' Actually does the job with the arguments provided. - - :arg request: the request that was called that we want to add JSONP - support to - :type request: aiohttp.web_request.Request - - ''' - response = yield from function(request, *args, **kwargs) - url_arg = request.query - callback = url_arg.get('callback') - if callback and request.method == 'GET': - if isinstance(callback, list): - callback = callback[0] - response.mimetype = 'application/javascript' - response.content_type = 'application/javascript' - response.text = f'{callback}({response.text});' - - return response - - return wrapper - - -@asyncio.coroutine -def _get_pkg(branch, name=None, action=None, srcname=None): +async def _get_pkg(branch, name=None, action=None, srcname=None): ''' Return the pkg information for the given package in the specified branch or raise an aiohttp exception. ''' @@ -95,53 +71,45 @@ def _get_pkg(branch, name=None, action=None, srcname=None): pkg = None wrongdb = False for repotype in ['updates-testing', 'updates', 'testing', None]: - - if repotype: - dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}-{repotype}-primary.sqlite' - else: - dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}-primary.sqlite' + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}'\ + '-primary.sqlite' if not os.path.exists(dbfile): wrongdb = True continue wrongdb = False - - session = yield from mdapilib.create_session(f'sqlite:///{dbfile}') - if name: + async with aiosqlite.connect(f'{dbfile}') as db: if action: - pkg = yield from mdapilib.get_package_by( - session, action, name) + # It is safe to format the query since the action does not come from the + # user. + query = GET_PACKAGE_BY.format(action) + async with db.execute(query, (name,)) as cursor: + pkg = await cursor.fetchall() + if pkg: + pkg = [Packages(*item) for item in pkg] + break + elif srcname: + async with db.execute(GET_PACKAGE_BY_SRC, (srcname+'%',)) as cursor: + pkg = await cursor.fetchone() + if pkg: + pkg = Packages(*pkg) + break else: - pkg = yield from mdapilib.get_package(session, name) - elif srcname: - pkg = yield from mdapilib.get_package_by_src(session, srcname) - session.close() - if pkg: - break - + async with db.execute(GET_PACKAGE, (name,)) as cursor: + pkg = await cursor.fetchone() + if pkg: + pkg = Packages(*pkg) + break if wrongdb: raise web.HTTPBadRequest() if not pkg: raise web.HTTPNotFound() - return (pkg, repotype) -def _get_pretty(request): - pretty = False - params = request.query - if params.get('pretty') in ['1', 'true']: - pretty = True - # Assume pretty if html is requested and pretty is not disabled - elif 'text/html' in request.headers.get('ACCEPT', ''): - pretty = True - return pretty - - -@asyncio.coroutine -def _expand_pkg_info(pkgs, branch, repotype=None): +async def _expand_pkg_info(pkgs, branch, repotype=None): ''' Return a JSON blob containing all the information we want to return for the provided package or packages. ''' @@ -152,286 +120,86 @@ def _expand_pkg_info(pkgs, branch, repotype=None): output = [] for pkg in pkgs: out = pkg.to_json() - dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}-primary.sqlite' - - session = yield from mdapilib.create_session(f'sqlite:///{dbfile}') - # Fill in some extra info - - # Basic infos, always present regardless of the version of the repo - for datatype in ['conflicts', 'obsoletes', 'provides', 'requires']: - data = yield from mdapilib.get_package_info( - session, pkg.pkgKey, datatype.capitalize()) - if data: - out[datatype] = [item.to_json() for item in data] + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}'\ + '-primary.sqlite' + + async with aiosqlite.connect(f'{dbfile}') as db: + # Fill in some extra info + # Basic infos, always present regardless of the version of the repo + for datatype in ['conflicts', + 'obsoletes', + 'provides', + 'requires', + 'enhances', + 'recommends', + 'suggests', + 'supplements']: + # It is safe to format the query since the datatype does not come from the + # user. + query = GET_PACKAGE_INFO.format(datatype) + async with db.execute(query, (pkg.pkgKey,)) as cursor: + data = await cursor.fetchall() + if data: + out[datatype] = [Dependencies(*item).to_json() for item in data] + else: + out[datatype] = data + + # Add the list of packages built from the same src.rpm + if pkg.rpm_sourcerpm: + async with db.execute(GET_CO_PACKAGE, (pkg.rpm_sourcerpm,)) as cursor: + copkgs = await cursor.fetchall() + out['co-packages'] = list(set([ + cpkg[2] for cpkg in copkgs + ])) else: - out[datatype] = data + out['co-packages'] = [] + out['repo'] = repotype if repotype else 'release' + output.append(out) - # New meta-data present for soft dependency management in RPM - for datatype in [ - 'enhances', 'recommends', 'suggests', 'supplements']: - data = yield from mdapilib.get_package_info( - session, pkg.pkgKey, datatype.capitalize()) - if data: - out[datatype] = [item.to_json() for item in data] - else: - out[datatype] = data - - # Add the list of packages built from the same src.rpm - if pkg.rpm_sourcerpm: - copkgs = yield from mdapilib.get_co_packages( - session, pkg.rpm_sourcerpm) - out['co-packages'] = list(set([ - cpkg.name for cpkg in copkgs - ])) - else: - out['co-packages'] = [] - out['repo'] = repotype if repotype else 'release' - session.close() - output.append(out) if singleton: return output[0] else: return output -@asyncio.coroutine -@allows_jsonp -def get_pkg(request): - _log.info(f'get_pkg {request}') - branch = request.match_info.get('branch') - pretty = _get_pretty(request) - name = request.match_info.get('name') - pkg, repotype = yield from _get_pkg(branch, name) - - output = yield from _expand_pkg_info(pkg, branch, repotype) - - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - output = web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - return output - - -@asyncio.coroutine -@allows_jsonp -def get_src_pkg(request): - _log.info(f'get_src_pkg {request}') - branch = request.match_info.get('branch') - pretty = _get_pretty(request) - name = request.match_info.get('name') - pkg, repotype = yield from _get_pkg(branch, srcname=name) - - output = yield from _expand_pkg_info(pkg, branch, repotype) - - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - - -@asyncio.coroutine -@allows_jsonp -def get_pkg_files(request): - _log.info(f'get_pkg_files {request}') - branch = request.match_info.get('branch') - name = request.match_info.get('name') - pretty = _get_pretty(request) - pkg, repotype = yield from _get_pkg(branch, name) - - dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}-filelists.sqlite' +async def _get_files(pkg_id, branch, repotype): + ''' Return the files list for the given package in the specified + branch. + ''' + dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}'\ + '-filelists.sqlite' if not os.path.exists(dbfile): raise web.HTTPBadRequest() - session2 = yield from mdapilib.create_session(f'sqlite:///{dbfile}') - filelist = yield from mdapilib.get_files(session2, pkg.pkgId) - session2.close() + async with aiosqlite.connect(f"{dbfile}") as db: + async with db.execute(GET_FILES, (pkg_id,)) as cursor: + filelists = await cursor.fetchall() + + filelists = [FileList(*item) for item in filelists] output = { - 'files': [fileinfo.to_json() for fileinfo in filelist], + 'files': [fileinfo.to_json() for fileinfo in filelists], 'repo': repotype if repotype else 'release', } - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - + return output -@asyncio.coroutine -@allows_jsonp -def get_pkg_changelog(request): - _log.info(f'get_pkg_changelog {request}') - branch = request.match_info.get('branch') - name = request.match_info.get('name') - pretty = _get_pretty(request) - pkg, repotype = yield from _get_pkg(branch, name) +async def _get_changelog(pkg_id, branch, repotype): + ''' Return the changelog for the given package in the specified + branch. + ''' dbfile = f'{CONFIG["DB_FOLDER"]}/mdapi-{branch}{"-"+repotype if repotype else ""}-other.sqlite' if not os.path.exists(dbfile): raise web.HTTPBadRequest() - session2 = yield from mdapilib.create_session(f'sqlite:///{dbfile}') - changelogs = yield from mdapilib.get_changelog(session2, pkg.pkgId) - session2.close() + async with aiosqlite.connect(f"{dbfile}") as db: + async with db.execute(GET_CHANGELOGS, (pkg_id,)) as cursor: + changelogs = await cursor.fetchall() + + changelogs = [ChangeLog(*item) for item in changelogs] output = { 'changelogs': [changelog.to_json() for changelog in changelogs], 'repo': repotype if repotype else 'release', } - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response( - body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - - -@asyncio.coroutine -def list_branches(request): - ''' Return the list of all branches currently supported by mdapi - ''' - _log.info(f'list_branches: {request}') - pretty = _get_pretty(request) - output = sorted(list(set([ - # Remove the front part `mdapi-` and the end part -.sqlite - filename.replace('mdapi-', '').rsplit('-', 2)[0].replace( - '-updates', '') - for filename in os.listdir(CONFIG['DB_FOLDER']) - if filename.startswith('mdapi') and filename.endswith('.sqlite') - ]))) - - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - response = web.Response(body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - - # The decorator doesn't work for this endpoint, so do it manually here - # I am not really sure what doesn't work but it seems this endpoint is - # returning an object instead of the expected generator despite it being - # flagged as an asyncio coroutine - url_arg = request.query - callback = url_arg.get('callback') - if callback and request.method == 'GET': - if isinstance(callback, list): - callback = callback[0] - response.mimetype = 'application/javascript' - response.content_type = 'application/javascript' - response.text = f'{callback}({response.text});' - - return response - - -@asyncio.coroutine -@allows_jsonp -def process_dep(request, action): - ''' Return the information about the packages having the specified - action (provides, requires, obsoletes...) - ''' - _log.info(f'process_dep {action}: {request}') - branch = request.match_info.get('branch') - pretty = _get_pretty(request) - name = request.match_info.get('name') - - try: - pkg, repotype = yield from _get_pkg(branch, name, action=action) - except: - raise web.HTTPBadRequest() - - output = yield from _expand_pkg_info(pkg, branch, repotype) - - args = {} - if pretty: - args = dict(sort_keys=True, indent=4, separators=(',', ': ')) - - return web.Response(body=json.dumps(output, **args).encode('utf-8'), - content_type='application/json') - - -@asyncio.coroutine -def get_provides(request): - return process_dep(request, 'provides') - - -@asyncio.coroutine -def get_requires(request): - return process_dep(request, 'requires') - - -@asyncio.coroutine -def get_obsoletes(request): - return process_dep(request, 'obsoletes') - - -@asyncio.coroutine -def get_conflicts(request): - return process_dep(request, 'conflicts') - - -@asyncio.coroutine -def get_enhances(request): - return process_dep(request, 'enhances') - - -@asyncio.coroutine -def get_recommends(request): - return process_dep(request, 'recommends') - - -@asyncio.coroutine -def get_suggests(request): - return process_dep(request, 'suggests') - - -@asyncio.coroutine -def get_supplements(request): - return process_dep(request, 'supplements') - - -@asyncio.coroutine -def index(request): - _log.info(f'index {request}') - return web.Response( - body=INDEX.encode('utf-8'), - content_type='text/html', - charset='utf-8') - - -def _set_routes(app): - routes = [] - prefix = CONFIG.get('PREFIX', '') - if prefix: - routes.append(('', index)) - - routes.extend([ - ('/', index), - ('/branches', list_branches), - ('/{branch}/pkg/{name}', get_pkg), - ('/{branch}/srcpkg/{name}', get_src_pkg), - - ('/{branch}/provides/{name}', get_provides), - ('/{branch}/requires/{name}', get_requires), - ('/{branch}/obsoletes/{name}', get_obsoletes), - ('/{branch}/conflicts/{name}', get_conflicts), - - ('/{branch}/enhances/{name}', get_enhances), - ('/{branch}/recommends/{name}', get_recommends), - ('/{branch}/suggests/{name}', get_suggests), - ('/{branch}/supplements/{name}', get_supplements), - - ('/{branch}/files/{name}', get_pkg_files), - ('/{branch}/changelog/{name}', get_pkg_changelog), - ]) - for route in routes: - app.router.add_route('GET', prefix + route[0], route[1]) - return app + return output diff --git a/mdapi/changelog.py b/mdapi/changelog.py deleted file mode 100644 index cc682d3..0000000 --- a/mdapi/changelog.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2015 Red Hat, Inc. -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. -# - -''' -DB mapping for the other sqlite DB. -''' - -import sqlalchemy as sa - -from sqlalchemy.ext.declarative import declarative_base - -BASE = declarative_base() - - -class Package(BASE): - ''' Maps the packages table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'packages' - pkgKey = sa.Column(sa.Integer, primary_key=True) - pkgId = sa.Column(sa.Text) - - -class Changelog(BASE): - ''' Maps the packages table in the filelists.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'changelog' - pkgKey = sa.Column(sa.Integer, primary_key=True) - author = sa.Column(sa.Text, primary_key=True) - changelog = sa.Column(sa.Text, primary_key=True) - date = sa.Column(sa.Integer, primary_key=True) - - def to_json(self): - filelist = { - 'author': self.author, - 'changelog': self.changelog, - 'date': self.date, - } - return filelist diff --git a/mdapi/db.py b/mdapi/db.py new file mode 100644 index 0000000..95c6d90 --- /dev/null +++ b/mdapi/db.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2019 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. You +# should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission +# of Red Hat, Inc. +# + +from dataclasses import dataclass + +GET_PACKAGE = """SELECT pkgKey, + pkgId, + name, + rpm_sourcerpm, + epoch, + version, + release, + arch, + summary, + description, + url + FROM packages + WHERE name = ? + ORDER BY epoch DESC, version DESC, release DESC""" + +GET_PACKAGE_INFO = """SELECT rowid, + pkgKey, + name, + epoch, + version, + release, + flags + FROM {} + WHERE pkgKey = ?""" + +GET_CO_PACKAGE = """SELECT pkgKey, + pkgId, + name, + rpm_sourcerpm, + epoch, + version, + release, + arch, + summary, + description, + url + FROM packages + WHERE rpm_sourcerpm = ?""" + +GET_PACKAGE_BY_SRC = """SELECT pkgKey, + pkgId, + name, + rpm_sourcerpm, + epoch, + version, + release, + arch, + summary, + description, + url + FROM packages + WHERE rpm_sourcerpm LIKE ? + ORDER BY epoch DESC, version DESC, release DESC""" + +GET_PACKAGE_BY = """SELECT p.pkgKey, + p.pkgId, + p.name, + p.rpm_sourcerpm, + p.epoch, + p.version, + p.release, + p.arch, + p.summary, + p.description, + p.url + FROM packages p + JOIN {} t ON t.pkgKey = p.pkgKey + WHERE t.name = ? + ORDER BY p.epoch DESC, p.version DESC, p.release DESC""" + +GET_FILES = """SELECT f.pkgKey, + f.dirname, + f.filenames, + f.filetypes + FROM filelist f + JOIN packages p ON p.pkgId = ? + WHERE f.pkgKey = p.pkgKey + ORDER BY f.filenames""" + + +GET_CHANGELOGS = """SELECT c.pkgKey, + c.author, + c.changelog, + c.date + FROM changelog c + JOIN packages p ON p.pkgId = ? + WHERE c.pkgKey = p.pkgKey + ORDER BY c.date DESC""" + + +@dataclass +class Packages: + pkgKey: int + pkgId: str + name: str + rpm_sourcerpm: str + epoch: str + version: str + release: str + arch: str + summary: str + description: str + url: str + + @property + def basename(self): + return self.rpm_sourcerpm.rsplit("-", 2)[0] + + def to_json(self): + pkg = { + 'arch': self.arch, + 'epoch': self.epoch, + 'version': self.version, + 'release': self.release, + 'summary': self.summary, + 'description': self.description, + 'basename': self.basename, + 'url': self.url, + } + return pkg + + +@dataclass +class Dependencies: + rowid: int + pkgKey: int + name: str + epoch: str + version: str + release: str + flags: str + + def to_json(self): + pkg = { + 'name': self.name, + 'epoch': self.epoch, + 'version': self.version, + 'release': self.release, + 'flags': self.flags, + } + return pkg + + +@dataclass +class FileList: + pkgKey: int + dirname: str + filenames: str + filetypes: str + + def to_json(self): + filelist = { + 'dirname': self.dirname, + 'filenames': self.filenames, + 'filetypes': self.filetypes, + } + return filelist + + +@dataclass +class ChangeLog: + pkgKey: int + author: str + changelog: str + date: int + + def to_json(self): + changelog = { + 'author': self.author, + 'changelog': self.changelog, + 'date': self.date, + } + return changelog diff --git a/mdapi/filelist.py b/mdapi/filelist.py deleted file mode 100644 index 6506a2d..0000000 --- a/mdapi/filelist.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2015 Red Hat, Inc. -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. -# - -''' -DB mapping for the filelists sqlite DB. -''' - -import sqlalchemy as sa - -from sqlalchemy.ext.declarative import declarative_base - -BASE = declarative_base() - - -class Package(BASE): - ''' Maps the packages table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'packages' - pkgKey = sa.Column(sa.Integer, primary_key=True) - pkgId = sa.Column(sa.Text) - - -class Filelist(BASE): - ''' Maps the packages table in the filelists.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'filelist' - pkgKey = sa.Column(sa.Integer, primary_key=True) - dirname = sa.Column(sa.Text, primary_key=True) - filenames = sa.Column(sa.Text, primary_key=True) - filetypes = sa.Column(sa.Text, primary_key=True) - - def to_json(self): - filelist = { - 'dirname': self.dirname, - 'filenames': self.filenames, - 'filetypes': self.filetypes, - } - return filelist diff --git a/mdapi/index.html b/mdapi/index.html index 549af20..44f0f0a 100644 --- a/mdapi/index.html +++ b/mdapi/index.html @@ -68,7 +68,7 @@ at: /branches - /branches + /branches Note: @@ -88,7 +88,7 @@ by querying: So for example, for the kernel in rawhide: - /rawhide/pkg/kernel + /rawhide/pkg/kernel You can also retrieve information about a specific package on a specific @@ -98,7 +98,7 @@ branch via the name of its source package by querying: So for example, for the python-natsort in rawhide that only exists as src.rpm: - /rawhide/srcpkg/python-natsort + /rawhide/srcpkg/python-natsort Retrieve the list of files in a package @@ -111,7 +111,7 @@ specific branch by querying: So for example, for the kernel-core in rawhide: - /rawhide/files/kernel-core + /rawhide/files/kernel-core Retrieve the changelog of a package @@ -124,7 +124,7 @@ by querying: So for example, for the kernel in rawhide: - /rawhide/changelog/kernel + /rawhide/changelog/kernel Retrieve the packages having a specific property @@ -143,29 +143,29 @@ For example to retrieve the list of packages that require a specific package: Few examples: packages requiring R in rawhide: - /rawhide/requires/R + /rawhide/requires/R To see what R itself requires, check its information using: /rawhide/pkg/R packages providing perl(SetupLog) in rawhide: - /rawhide/provides/perl(SetupLog) + /rawhide/provides/perl(SetupLog) packages obsoleting cabal2spec in rawhide: - rawhide/obsoletes/cabal2spec + rawhide/obsoletes/cabal2spec packages conflicting with mariadb in rawhide: - rawhide/conflicts/mariadb + rawhide/conflicts/mariadb packages enhancing httpd in rawhide: - rawhide/enhances/httpd + rawhide/enhances/httpd packages recommending flac in rawhide: - rawhide/recommends/flac + rawhide/recommends/flac packages suggesting R-tools in rawhide: - rawhide/suggests/R-tools + rawhide/suggests/R-tools packages supplementing `(hunspell and langpacks-fr)` in rawhide: - rawhide/supplements/(hunspell and langpacks-fr) + rawhide/supplements/(hunspell and langpacks-fr) diff --git a/mdapi/lib.py b/mdapi/lib.py deleted file mode 100644 index 2af05ab..0000000 --- a/mdapi/lib.py +++ /dev/null @@ -1,276 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2015 Red Hat, Inc. -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. -# - -''' -MDAPI internal API to interact with the database. -''' - -import contextlib -import time - -import asyncio -import sqlalchemy as sa - -from sqlalchemy.orm import sessionmaker -from sqlalchemy.orm import scoped_session -from sqlalchemy.exc import SQLAlchemyError, OperationalError - -import mdapi.changelog as changelog -import mdapi.filelist as filelist -import mdapi.primary as primary - - -RETRY_ATTEMPT = 3 - - -@asyncio.coroutine -def create_session(db_url, debug=False, pool_recycle=3600): - """ Create the Session object to use to query the database. - - :arg db_url: URL used to connect to the database. The URL contains - information with regards to the database engine, the host to connect - to, the user and password and the database name. - ie: ://:@/ - :kwarg debug: a boolean specifying wether we should have the verbose - output of sqlalchemy or not. - :return a Session that can be used to query the database. - - """ - engine = sa.create_engine( - db_url, echo=debug, pool_recycle=pool_recycle) - scopedsession = scoped_session(sessionmaker(bind=engine)) - return scopedsession - - -@contextlib.contextmanager -def session_manager(db_url, debug=False, pool_recycle=3600): - """ A handy context manager for our sessions. """ - session = yield from create_session( - db_url, debug=debug, pool_recycle=pool_recycle) - try: - yield session - session.commit() - except: - session.rollback() - raise - finally: - session.close() - - -@asyncio.coroutine -def get_package(session, pkg_name): - ''' Return information about a package, if we can find it. - ''' - output = None - cnt = 0 - try: - pkg = session.query( - primary.Package - ).filter( - primary.Package.name == pkg_name - ).order_by( - primary.Package.epoch.desc(), - primary.Package.version.desc(), - primary.Package.release.desc(), - ) - output = pkg.first() - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - output = yield from get_package(session, pkg_name) - - return output - - -@asyncio.coroutine -def get_package_by_src(session, pkg_name): - ''' Return information about a package, if we can find it. - ''' - # First try if there is a package matching exactly the provided name - simple_match = yield from get_package(session, pkg_name) - if simple_match and simple_match.basename == pkg_name: - return simple_match - - # If there is not a direct match, look by the sourcerpm name - cnt = 0 - try: - pkg = session.query( - primary.Package - ).filter( - primary.Package.rpm_sourcerpm.like('{}%'.format(pkg_name)) - ).order_by( - primary.Package.epoch.desc(), - primary.Package.version.desc(), - primary.Package.release.desc(), - ) - for pkg in pkg.all(): - if pkg.basename == pkg_name: - return pkg - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - yield from get_package_by_src(session, pkg_name) - - -@asyncio.coroutine -def get_package_by(session, tablename, key, cnt=None): - ''' Return information the package providing the provides, if we can find it. - ''' - table = getattr(primary, tablename.capitalize()) - - output = None - cnt = cnt or 0 - try: - pkg = session.query( - primary.Package - ).filter( - table.name == key - ).filter( - table.pkgKey == primary.Package.pkgKey - ).order_by( - primary.Package.epoch.desc(), - primary.Package.version.desc(), - primary.Package.release.desc(), - ) - output = pkg.all() - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - output = yield from get_package_by( - session, tablename, key, cnt=cnt) - - return output - - -@asyncio.coroutine -def get_package_info(session, pkgKey, tablename): - ''' Return the information contained in the specified table for the - given package. - ''' - table = getattr(primary, tablename) - output = None - cnt = 0 - try: - query = session.query( - table - ).filter( - table.pkgKey == pkgKey - ) - output = query.all() - except OperationalError: - return None - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - output = yield from get_package_info(session, pkgKey, tablename) - - return output - - -@asyncio.coroutine -def get_co_packages(session, srcpkg_name): - ''' Return the name of all the packages coming from the same - source-package. - ''' - output = None - cnt = 0 - try: - pkg = session.query( - primary.Package - ).filter( - primary.Package.rpm_sourcerpm == srcpkg_name - ) - output = pkg.all() - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - output = yield from get_co_packages(session, srcpkg_name) - - return output - - -@asyncio.coroutine -def get_files(session, pkg_id): - ''' Return the list of all the files in a package given its key. - ''' - output = None - cnt = 0 - try: - pkg = session.query( - filelist.Filelist - ).filter( - filelist.Package.pkgId == pkg_id, - filelist.Filelist.pkgKey == filelist.Package.pkgKey - ).order_by( - filelist.Filelist.filenames - ) - output = pkg.all() - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - output = yield from get_files(session, pkg_id) - - return output - - -@asyncio.coroutine -def get_changelog(session, pkg_id): - ''' Return the list of all the changelog in a package given its key. - ''' - output = None - cnt = 0 - try: - pkg = session.query( - changelog.Changelog - ).filter( - changelog.Package.pkgId == pkg_id, - changelog.Changelog.pkgKey == changelog.Package.pkgKey - ).order_by( - changelog.Changelog.date.desc() - ) - output = pkg.all() - except SQLAlchemyError as err: - cnt += 1 - if cnt > RETRY_ATTEMPT: - raise - else: - time.sleep(0.1) - output = yield from get_changelog(session, pkg_id) - - return output diff --git a/mdapi/primary.py b/mdapi/primary.py deleted file mode 100644 index ed27e47..0000000 --- a/mdapi/primary.py +++ /dev/null @@ -1,151 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © 2015 Red Hat, Inc. -# -# This copyrighted material is made available to anyone wishing to use, -# modify, copy, or redistribute it subject to the terms and conditions -# of the GNU General Public License v.2, or (at your option) any later -# version. This program is distributed in the hope that it will be -# useful, but WITHOUT ANY WARRANTY expressed or implied, including the -# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR -# PURPOSE. See the GNU General Public License for more details. You -# should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, -# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# Any Red Hat trademarks that are incorporated in the source -# code or documentation are not subject to the GNU General Public -# License and may only be used or replicated with the express permission -# of Red Hat, Inc. -# - -''' -DB mapping for the primary sqlite DB. -''' - -import sqlalchemy as sa - -from sqlalchemy.ext.declarative import declarative_base - -BASE = declarative_base() - - -class Package(BASE): - ''' Maps the packages table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'packages' - pkgKey = sa.Column(sa.Integer, primary_key=True) - pkgId = sa.Column(sa.Text) - name = sa.Column(sa.Text) - rpm_sourcerpm = sa.Column(sa.Text) - epoch = sa.Column(sa.Text) - version = sa.Column(sa.Text) - release = sa.Column(sa.Text) - arch = sa.Column(sa.Text) - summary = sa.Column(sa.Text) - description = sa.Column(sa.Text) - url = sa.Column(sa.Text) - - @property - def basename(self): - ''' Return the base package name using the rpm_sourcerpms info. ''' - return self.rpm_sourcerpm.rsplit('-', 2)[0] - - def to_json(self): - pkg = { - 'arch': self.arch, - 'epoch': self.epoch, - 'version': self.version, - 'release': self.release, - 'summary': self.summary, - 'description': self.description, - 'basename': self.basename, - 'url': self.url, - } - return pkg - - -class BaseDependency(object): - ''' Base mapping for the tables in the primary.sqlite database that - contain all the dependencies information - ''' - rowid = sa.Column(sa.Integer, primary_key=True) - pkgKey = sa.Column(sa.Integer, index=True) - name = sa.Column(sa.Text) - epoch = sa.Column(sa.Text) - version = sa.Column(sa.Text) - release = sa.Column(sa.Text) - flags = sa.Column(sa.Text) - - def to_json(self): - pkg = { - 'name': self.name, - 'epoch': self.epoch, - 'version': self.version, - 'release': self.release, - 'flags': self.flags, - } - return pkg - - -BASEDEP = declarative_base(cls=BaseDependency) - - -class Requires(BASEDEP): - ''' Maps the requires table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'requires' - - -class Provides(BASEDEP): - ''' Maps the provides table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'provides' - - -class Conflicts(BASEDEP): - ''' Maps the conflicts table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'conflicts' - - -class Obsoletes(BASEDEP): - ''' Maps the provides table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'obsoletes' - - -# New soft-dependencies - - -class Enhances(BASEDEP): - ''' Maps the enhances table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'enhances' - - -class Recommends(BASEDEP): - ''' Maps the recommends table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'recommends' - - -class Suggests(BASEDEP): - ''' Maps the suggests table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'suggests' - - -class Supplements(BASEDEP): - ''' Maps the supplements table in the primary.sqlite database from - repodata to a python object. - ''' - __tablename__ = 'supplements' diff --git a/mdapi/server.py b/mdapi/server.py index bdc9ca1..514a253 100644 --- a/mdapi/server.py +++ b/mdapi/server.py @@ -3,18 +3,53 @@ import logging.config from aiohttp import web -from mdapi import CONFIG, _set_routes - - -def main(): +from mdapi import CONFIG +from mdapi.views import ( + index, + list_branches, + get_pkg, + get_src_pkg, + get_provides, + get_requires, + get_obsoletes, + get_conflicts, + get_enhances, + get_recommends, + get_suggests, + get_supplements, + get_pkg_files, + get_pkg_changelog +) + + +async def init_app(): + """ Creates the aiohttp application. + This function creates a web application configure the routes and + returns the application object.""" logging.basicConfig() logging.config.dictConfig(CONFIG.get("LOGGING") or {"version": 1}) app = web.Application() - app = _set_routes(app) - host = CONFIG.get("HOST", "127.0.0.1") - port = CONFIG.get("PORT", 8080) + app.add_routes([ + web.get('/', index), + web.get('/branches', list_branches), + web.get('/{branch}/pkg/{name}', get_pkg), + web.get('/{branch}/srcpkg/{name}', get_src_pkg), + + web.get('/{branch}/provides/{name}', get_provides), + web.get('/{branch}/requires/{name}', get_requires), + web.get('/{branch}/obsoletes/{name}', get_obsoletes), + web.get('/{branch}/conflicts/{name}', get_conflicts), + + web.get('/{branch}/enhances/{name}', get_enhances), + web.get('/{branch}/recommends/{name}', get_recommends), + web.get('/{branch}/suggests/{name}', get_suggests), + web.get('/{branch}/supplements/{name}', get_supplements), + + web.get('/{branch}/files/{name}', get_pkg_files), + web.get('/{branch}/changelog/{name}', get_pkg_changelog), + ]) - web.run_app(app, host=host, port=port) + return app diff --git a/mdapi/views.py b/mdapi/views.py new file mode 100644 index 0000000..042b931 --- /dev/null +++ b/mdapi/views.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# +# Copyright © 2019 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions +# of the GNU General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. You +# should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission +# of Red Hat, Inc. +# + +import logging +import os + +from aiohttp import web + +from mdapi import _get_pkg, _expand_pkg_info, _get_files, _get_changelog, CONFIG + +indexfile = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'index.html') + + +_log = logging.getLogger(__name__) + + +async def index(request): + _log.info(f'index {request}') + return web.FileResponse(indexfile) + + +async def get_pkg(request): + _log.info(f'get_pkg {request}') + branch = request.match_info.get('branch') + name = request.match_info.get('name') + pkg, repotype = await _get_pkg(branch, name) + + output = await _expand_pkg_info(pkg, branch, repotype) + + return web.json_response(output) + + +async def get_src_pkg(request): + _log.info(f'get_src_pkg {request}') + branch = request.match_info.get('branch') + name = request.match_info.get('name') + pkg, repotype = await _get_pkg(branch, srcname=name) + + output = await _expand_pkg_info(pkg, branch, repotype) + + return web.json_response(output) + + +async def list_branches(request): + ''' Return the list of all branches currently supported by mdapi + ''' + _log.info(f'list_branches: {request}') + output = sorted(list(set([ + # Remove the front part `mdapi-` and the end part -.sqlite + filename.replace('mdapi-', '').rsplit('-', 2)[0].replace( + '-updates', '') + for filename in os.listdir(CONFIG['DB_FOLDER']) + if filename.startswith('mdapi') and filename.endswith('.sqlite') + ]))) + + return web.json_response(output) + + +async def _process_dep(request, action): + ''' Return the information about the packages having the specified + action (provides, requires, obsoletes...) + ''' + _log.info(f'process_dep {action}: {request}') + branch = request.match_info.get('branch') + name = request.match_info.get('name') + + try: + pkg, repotype = await _get_pkg(branch, name, action=action) + except: + raise web.HTTPBadRequest() + + output = await _expand_pkg_info(pkg, branch, repotype) + + return web.json_response(output) + + +async def get_provides(request): + return await _process_dep(request, 'provides') + + +async def get_requires(request): + return await _process_dep(request, 'requires') + + +async def get_obsoletes(request): + return await _process_dep(request, 'obsoletes') + + +async def get_conflicts(request): + return await _process_dep(request, 'conflicts') + + +async def get_enhances(request): + return await _process_dep(request, 'enhances') + + +async def get_recommends(request): + return await _process_dep(request, 'recommends') + + +async def get_suggests(request): + return await _process_dep(request, 'suggests') + + +async def get_supplements(request): + return await _process_dep(request, 'supplements') + + +async def get_pkg_files(request): + _log.info(f'get_pkg_files {request}') + branch = request.match_info.get('branch') + name = request.match_info.get('name') + pkg, repotype = await _get_pkg(branch, name) + output = await _get_files(pkg.pkgId, branch, repotype) + + return web.json_response(output) + + +async def get_pkg_changelog(request): + _log.info(f'get_pkg_changelog {request}') + branch = request.match_info.get('branch') + name = request.match_info.get('name') + pkg, repotype = await _get_pkg(branch, name) + output = await _get_changelog(pkg.pkgId, branch, repotype) + + return web.json_response(output) diff --git a/requirements.txt b/requirements.txt index 345bfbf..9549806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ aiohttp >= 3.5.4 +aiosqlite fedora_messaging # this is a requirement of aiohttp but better safe than sorry requests -sqlalchemy werkzeug -flufl.lock +uvloop +gunicorn diff --git a/tests/test_mdapi_data.py b/tests/test_mdapi_data.py index a815cc4..c9fc295 100644 --- a/tests/test_mdapi_data.py +++ b/tests/test_mdapi_data.py @@ -41,6 +41,8 @@ sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) import mdapi +from mdapi.server import init_app + HERE = os.path.join(os.path.dirname(os.path.abspath(__file__))) @@ -79,11 +81,11 @@ def tmpdir(): @pytest.fixture -def cli(set_env, tmpdir, loop, aiohttp_client): +async def cli(set_env, tmpdir, loop, test_client): mdapi.CONFIG['DB_FOLDER'] = tmpdir - app = web.Application() - app = mdapi._set_routes(app) - return loop.run_until_complete(aiohttp_client(app)) + app = await init_app() + return await test_client(app) + async def test_view_index_page(cli): diff --git a/tests/test_mdapi_empty.py b/tests/test_mdapi_empty.py index 92f3160..7c3de02 100644 --- a/tests/test_mdapi_empty.py +++ b/tests/test_mdapi_empty.py @@ -38,17 +38,16 @@ sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) import mdapi -import mdapi.lib +from mdapi.server import init_app HERE = os.path.join(os.path.dirname(os.path.abspath(__file__))) @pytest.fixture -def cli(loop, aiohttp_client): +async def cli(loop, test_client): mdapi.CONFIG['DB_FOLDER'] = '.' - app = web.Application() - app = mdapi._set_routes(app) - return loop.run_until_complete(aiohttp_client(app)) + app = await init_app() + return await test_client(app) async def test_view_index_page(cli):