From 28faacde9146b0cd471c31da9438bb90552b1b9c Mon Sep 17 00:00:00 2001 From: Bojan Jovanović Date: May 13 2019 12:13:51 +0000 Subject: initial commit for proof-of-concept --- diff --git a/proof-of-concept/data/rhbzsec.db b/proof-of-concept/data/rhbzsec.db new file mode 100644 index 0000000..31d5b2a Binary files /dev/null and b/proof-of-concept/data/rhbzsec.db differ diff --git a/proof-of-concept/example1.py b/proof-of-concept/example1.py new file mode 100644 index 0000000..3c1da57 --- /dev/null +++ b/proof-of-concept/example1.py @@ -0,0 +1,27 @@ +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Sofwtare Foundation; either version 2 of the Licens, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.htm for +# the full text of the license + +# Seems as peewee is good solution for database wrapper + +from peewee import chunked +from model import rhbug +import model +import rhbz_consumer + +# SQLITE_MAX_VARIABLE_NUMBER = 50000 +# Populate the data from json response +# Clever solution would be to find a differences +# Hacky workaround +rhbug.drop_table() +rhbug.create_table() + +data = rhbz_consumer.get_bugs_dict() +# SQLite have a limit about number of record for bulk insert +# this is a workaround +with model.db.atomic(): + for batch in chunked(data, 100): + rhbug.insert_many(batch).execute() + diff --git a/proof-of-concept/mockup-images/rhbzbarchart.png b/proof-of-concept/mockup-images/rhbzbarchart.png new file mode 100644 index 0000000..3b468a2 Binary files /dev/null and b/proof-of-concept/mockup-images/rhbzbarchart.png differ diff --git a/proof-of-concept/requirements.txt b/proof-of-concept/requirements.txt new file mode 100644 index 0000000..41dc30a --- /dev/null +++ b/proof-of-concept/requirements.txt @@ -0,0 +1,5 @@ +matplotlib==3.0.3 +numpy==1.16.3 +pandas==0.24.2 +peewee==3.9.5 +requests==2.21.0 diff --git a/proof-of-concept/rhbz_consumer.py b/proof-of-concept/rhbz_consumer.py new file mode 100644 index 0000000..a807a9e --- /dev/null +++ b/proof-of-concept/rhbz_consumer.py @@ -0,0 +1,63 @@ +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Sofwtare Foundation; either version 2 of the Licens, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.htm for +# the full text of the license + +import json +import requests + +URL = "https://bugzilla.redhat.com/rest/bug?bug_status=NEW&bug_status=ASSIGNED&classification=Fedora&j_top=OR&keywords=SecurityTracking%2C%20&keywords_type=allwords&known_name=fed-mod-crit&limit=0&list_id=9978724&order=priority%2Cbug_id&product=Fedora&query_format=advanced&v1=high&v2=urgent" + + +def get_bugs(): + rows = [] + try: + resp = requests.get(URL,timeout=50) + resp.raise_for_status() + except requests.exceptions.HTTPError as errh: + print('HTTP Error:', errh) + except requests.exceptions.ConnectionError as errc: + print('Error Connecting:', errc) + except requests.exceptions.Timeout as errt: + print('Timeout Error:', errt) + except requests.exceptions.RequestException as err: + print('Oops: Something else:', err) + else: + jbugs = json.loads(resp.text) + print(type(jbugs)) + for b in jbugs['bugs']: + rows.append((b['id'], b['creation_time'], b['component'][0], b['summary'], b['version'][0])) + finally: + return rows + +# peewee expect dictation for bulk insert +def get_bugs_dict(): + rhrows = [] + try: + resp = requests.get(URL,timeout=50) + resp.raise_for_status() + except requests.exceptions.HTTPError as errh: + print('HTTP Error:', errh) + except requests.exceptions.ConnectionError as errc: + print('Error Connecting:', errc) + except requests.exceptions.Timeout as errt: + print('Timeout Error:', errt) + except requests.exceptions.RequestException as err: + print('Oops: Something else:', err) + else: + jbugs = json.loads(resp.text) + + for b in jbugs['bugs']: + rhrow = {} + # rows.append((b['id'], b['creation_time'], b['component'][0], b['summary'], b['version'][0])) + rhrow['rhbzid'] = b['id'] + rhrow['creation_time'] = b['creation_time'] + rhrow['component'] = b['component'][0] + rhrow['summary'] = b['summary'] + rhrow['version'] = b['version'] + rhrows.append(rhrow) + finally: + return rhrows + + diff --git a/proof-of-concept/rhbzbarchart.py b/proof-of-concept/rhbzbarchart.py new file mode 100644 index 0000000..6d5c7be --- /dev/null +++ b/proof-of-concept/rhbzbarchart.py @@ -0,0 +1,47 @@ +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Sofwtare Foundation; either version 2 of the Licens, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.htm for +# the full text of the license + +# Bar chart for top 10 packages per number of bugs +# ----------------- query ------------------------ +# select count(rhbzid) as topof, component from rhbug +# group by component +# order by topof DESC limit 10; +# ------------------------------------------------ +import os +import model +import matplotlib.pyplot as plt +from peewee import fn +import pandas +import numpy + +query = (model.rhbug + .select(fn.COUNT(model.rhbug.rhbzid).alias('topof'),model.rhbug.component) + .group_by(model.rhbug.component) + .order_by(fn.COUNT(model.rhbug.rhbzid).desc()) + .limit(10) + ) + +df = pandas.DataFrame(list(query.dicts())) +data = df.to_numpy() + +plt.rcdefaults() + +component = [s for (s, num) in data] +x_pos = [i for i, _ in enumerate(component)] +y_pos = numpy.arange(len(component)) +numofbug = [ num for (s, num) in data] + + +plt.barh(component, numofbug, 0.75) +plt.ylabel("Component") +plt.xlabel("Number of bugs") +for i, v in enumerate(numofbug): + plt.text(v, i, " " + str(v), va='center') + +# plt.show() +fig_path = os.path.join(os.path.curdir, 'mockup-images', 'rhbzbarchart.png') +plt.savefig(os.path.join(fig_path), dpi=300, format='png', bbox_inches='tight') +