| |
@@ -0,0 +1,340 @@
|
| |
+ #
|
| |
+ # dashboard.py - Utilities to parse data sources for packager dashboard
|
| |
+ #
|
| |
+ # Copyright 2020, Red Hat, Inc
|
| |
+ #
|
| |
+ # This program is free software; you can redistribute it and/or modify
|
| |
+ # it under the terms of the GNU General Public License as published by
|
| |
+ # the Free Software Foundation; either version 2 of the License, or
|
| |
+ # (at your option) any later version.
|
| |
+ #
|
| |
+ # This program is distributed in the hope that it will be useful,
|
| |
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
|
| |
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
| |
+ # GNU General Public License for more details.
|
| |
+ #
|
| |
+ # You should have received a copy of the GNU General Public License along
|
| |
+ # with this program; if not, write to the Free Software Foundation, Inc.,
|
| |
+ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
| |
+ #
|
| |
+ # Authors:
|
| |
+ # Frantisek Zatloukal <fzatlouk@redhat.com>
|
| |
+ # Josef Skladanka <jskladan@redhat.com>
|
| |
+
|
| |
+ import json
|
| |
+ import datetime
|
| |
+ import requests
|
| |
+ import hawkey
|
| |
+ import bugzilla
|
| |
+
|
| |
+ from hawkey._hawkey import ValueException
|
| |
+ from bodhi.client.bindings import BodhiClient
|
| |
+ from urllib3.util.retry import Retry
|
| |
+ from requests.adapters import HTTPAdapter
|
| |
+
|
| |
+ from oraculum import app, db
|
| |
+ from oraculum.models.dashboard_users import DashboardUserData
|
| |
+
|
| |
+ def update_user_access_time(user):
|
| |
+ """
|
| |
+ Updates user last_accessed with current timestamp
|
| |
+ """
|
| |
+ row = DashboardUserData.query.filter_by(username=user).first()
|
| |
+ if not row:
|
| |
+ row = DashboardUserData(user, datetime.datetime.utcnow())
|
| |
+ db.session.add(row)
|
| |
+ else:
|
| |
+ row.last_accessed = datetime.datetime.utcnow()
|
| |
+ db.session.commit()
|
| |
+
|
| |
+ def get_users_for_sync():
|
| |
+ """
|
| |
+ Returns list of usernames to be included in sync if conditions specified in settings.py are met
|
| |
+ """
|
| |
+ now = datetime.datetime.utcnow() - datetime.timedelta(days=app.config['ACTIVITY_REQUIRED'])
|
| |
+ row = DashboardUserData.query.filter(DashboardUserData.last_accessed >= now)
|
| |
+ return [user.username for user in row]
|
| |
+
|
| |
+ def name_in_nevra(name, nevra):
|
| |
+ """
|
| |
+ Checks if name/nevra matches
|
| |
+ """
|
| |
+ # Work around package naming bug in known "bad" packages
|
| |
+ if "fedora-obsolete-packages" in nevra:
|
| |
+ if "fedora-obsolete-packages" in name:
|
| |
+ return True
|
| |
+ return False
|
| |
+ if "epel-rpm-macros" in nevra:
|
| |
+ if "epel-rpm-macros" in name:
|
| |
+ return True
|
| |
+ return False
|
| |
+
|
| |
+ try:
|
| |
+ if name == hawkey.split_nevra(nevra).name:
|
| |
+ return True
|
| |
+ else:
|
| |
+ return False
|
| |
+ except ValueException:
|
| |
+ app.logger.error("Hawkey/bodhi issue, skipping name/nevra compare, FIX THIS!!! (%s, %s)" % (name, nevra))
|
| |
+ return False
|
| |
+
|
| |
+ def release_from_nevra(nevra):
|
| |
+ """
|
| |
+ Returns fcXX from nevra
|
| |
+ """
|
| |
+ split = nevra.split(".")[-1]
|
| |
+ if len(split) > 1:
|
| |
+ return split
|
| |
+ return "Unknown Release"
|
| |
+
|
| |
+ def process_update(update):
|
| |
+ """
|
| |
+ Cleans up single update dictionary to contain only data frontend needs
|
| |
+ """
|
| |
+ return {
|
| |
+ "pretty_name": update["title"],
|
| |
+ "updateid": update["alias"],
|
| |
+ "submission_date": update["date_submitted"],
|
| |
+ "release": release_from_nevra(update["title"]),
|
| |
+ "url": update["url"],
|
| |
+ "status": update["status"],
|
| |
+ "karma": update["karma"],
|
| |
+ "comments": len(update["comments"])
|
| |
+ }
|
| |
+
|
| |
+ def process_override(override):
|
| |
+ """
|
| |
+ Cleans up single override dictionary to contain only data frontend needs
|
| |
+ """
|
| |
+ return {
|
| |
+ "pretty_name": override["nvr"],
|
| |
+ "url": "https://bodhi.fedoraproject.org/overrides/" + override["nvr"],
|
| |
+ "submission_date": override["submission_date"],
|
| |
+ "expiration_date": override["expiration_date"],
|
| |
+ "release": release_from_nevra(override["nvr"])
|
| |
+ }
|
| |
+
|
| |
+ def get_orphans(packages, orphans_json):
|
| |
+ """
|
| |
+ Returns dict of dictionaries (all packages), where orphanned package has orphanned == True
|
| |
+ """
|
| |
+ orphans = {}
|
| |
+ for package in packages:
|
| |
+ orphans[package] = {
|
| |
+ "orphanned": False,
|
| |
+ "orphanned_since": None
|
| |
+ }
|
| |
+ if package in orphans_json:
|
| |
+ orphans[package] = {
|
| |
+ "orphanned": True,
|
| |
+ "orphanned_since": orphans_json[package]
|
| |
+ }
|
| |
+ return orphans
|
| |
+
|
| |
+ def get_json(json_url):
|
| |
+ """
|
| |
+ Returns json data from provided url
|
| |
+ """
|
| |
+ session = requests.Session()
|
| |
+ retries = Retry(total=5,
|
| |
+ backoff_factor=0.1,
|
| |
+ status_forcelist=[ 500, 502, 503, 504 ])
|
| |
+ session.mount('https://', HTTPAdapter(max_retries=retries))
|
| |
+ resp = session.get(json_url)
|
| |
+ return json.loads(resp.text)
|
| |
+
|
| |
+
|
| |
+ def get_pagure_groups():
|
| |
+ """
|
| |
+ Returns dictionary mapping all packager groups in pagure, it's members and packages
|
| |
+ "group_name" : {
|
| |
+ "users": [user_a, ...],
|
| |
+ "packages": [package_a, ...]
|
| |
+ }
|
| |
+ """
|
| |
+ groups_users = {}
|
| |
+ resp = get_json("https://src.fedoraproject.org/api/0/groups?per_page=100") # TODO: Hanlde pagination properly
|
| |
+
|
| |
+ for group in resp["groups"]:
|
| |
+ app.logger.debug("Checking out Pagure group %s" % group)
|
| |
+ group_resp = get_json("https://src.fedoraproject.org/api/0/group/%s?projects=1&acl=commit" % group)
|
| |
+ try:
|
| |
+ groups_users[group] = {
|
| |
+ "users": group_resp["members"],
|
| |
+ "packages": [project["name"] for project in group_resp["projects"]]
|
| |
+ }
|
| |
+ except TypeError:
|
| |
+ app.logger.error("Skipped Pagure group %s because of an error" % group)
|
| |
+ continue
|
| |
+ return groups_users
|
| |
+
|
| |
+ def get_user_group_packages(user, groups_map):
|
| |
+ """
|
| |
+ Returns list of packages user owns through a group
|
| |
+ """
|
| |
+ group_packages = []
|
| |
+ for group in groups_map:
|
| |
+ if user in groups_map[group]["users"]:
|
| |
+ group_packages.extend(groups_map[group]["packages"])
|
| |
+ return group_packages
|
| |
+
|
| |
+ def get_packages(user, pkg_owners_map, groups_map):
|
| |
+ """
|
| |
+ Returns all packages owned by user (including those owned through a group)
|
| |
+ returns dict of lists: "combined" - returns all packages, owned both directly or through group
|
| |
+ "group" - returns packages owned only through group
|
| |
+ "primary" - returns packages owned only directly
|
| |
+ """
|
| |
+ # packages_owners_map['rpms']['some_package_name'] contains list of 'some_package_name' maintainers
|
| |
+ if user == "orphan": # blacklist orphan user which has lots of unnecessary packages and would choke up our servers
|
| |
+ return {
|
| |
+ "primary": [],
|
| |
+ "group": [],
|
| |
+ "combined": []
|
| |
+ }
|
| |
+ packages = {}
|
| |
+ packages["primary"] = [package for package in pkg_owners_map['rpms'] if user in pkg_owners_map['rpms'][package]]
|
| |
+ packages["group"] = get_user_group_packages(user, groups_map)
|
| |
+ packages["combined"] = packages["primary"].copy()
|
| |
+ packages["combined"].extend(x for x in packages["group"] if x not in packages["primary"])
|
| |
+ return packages
|
| |
+
|
| |
+ def get_updates(packages, raw_updates):
|
| |
+ """
|
| |
+ Gets list of user owned packages and libkarma dump of all updates
|
| |
+ Returns dict of user owned packages and updates for them
|
| |
+ package_name: {
|
| |
+ "pretty_name": update["title"],
|
| |
+ "updateid": update["alias"],
|
| |
+ "submission_date": update["date_submitted"],
|
| |
+ "release": release_from_nevra(update["title"]),
|
| |
+ "url": update["url"],
|
| |
+ "status": update["status"],
|
| |
+ "karma": update["karma"],
|
| |
+ "comments": len(update["comments"])
|
| |
+ }
|
| |
+ """
|
| |
+ if len(packages) == 0:
|
| |
+ return {}
|
| |
+ data = {}
|
| |
+ # Prepare dict items for each package
|
| |
+ for package in packages:
|
| |
+ data[package] = []
|
| |
+
|
| |
+ # raw_updates["Fxx"][0...n]["builds"][0...n]["nvr"] = 'package_name-version-release'
|
| |
+ # raw_updates["Fxx"][0...n]["alias"] = FEDORA-YYYY-UPDATE_ID
|
| |
+ for release in raw_updates:
|
| |
+ for package in packages:
|
| |
+ present_updates = set() # helper set to deduplicate updates containing more than one package
|
| |
+ for update in raw_updates[release]:
|
| |
+ for build in update["builds"]:
|
| |
+ if name_in_nevra(package, build["nvr"]) and update["alias"] not in present_updates:
|
| |
+ data[package].append(process_update(update))
|
| |
+ present_updates.add(update["alias"])
|
| |
+
|
| |
+ return data
|
| |
+
|
| |
+ def get_package_prs(package):
|
| |
+ """
|
| |
+ Returns all open Pull Requests for a single package
|
| |
+ """
|
| |
+ data = []
|
| |
+ resp_package_prs = get_json("https://src.fedoraproject.org/api/0/rpms/%s/pull-requests" % package)
|
| |
+ if not "requests" in resp_package_prs:
|
| |
+ app.logger.error("Skipping PRs from package %s because Pagure returned invalid data" % package)
|
| |
+ return [] # Return early if pagure sent invalid data
|
| |
+ for request in resp_package_prs["requests"]:
|
| |
+ if request["status"] == "Open":
|
| |
+ data.append({
|
| |
+ "title": request["title"],
|
| |
+ "author": request["user"]['name'],
|
| |
+ "comments": len(request["comments"]),
|
| |
+ "date_created": str(datetime.datetime.fromtimestamp(int(request["date_created"]))),
|
| |
+ "last_updated": str(datetime.datetime.fromtimestamp(int(request["last_updated"]))),
|
| |
+ "ci_status": None, # Awaiting pagure api integration
|
| |
+ "url": "https://src.fedoraproject.org/rpms/%s/pull-request/%s" % (package, request["id"])
|
| |
+ })
|
| |
+ return data
|
| |
+
|
| |
+ def get_package_bugs(package):
|
| |
+ """
|
| |
+ Returns all open Bugs for a single package
|
| |
+ """
|
| |
+ data = []
|
| |
+ bzapi = bugzilla.Bugzilla("bugzilla.redhat.com")
|
| |
+
|
| |
+ query = bzapi.url_to_query("https://bugzilla.redhat.com/buglist.cgi?bug_status=NEW&bug_status=__open__&"
|
| |
+ "classification=Fedora&product=Fedora&component=%s" % package)
|
| |
+ query["include_fields"] = ["id", "summary", "version", "severity", "status", "last_change_time", "creation_time", "keywords"]
|
| |
+ bugs = bzapi.query(query)
|
| |
+ if len(bugs) == 0:
|
| |
+ return []
|
| |
+ for bug in bugs:
|
| |
+ data.append({
|
| |
+ "title": bug.summary,
|
| |
+ "bug_id": bug.id,
|
| |
+ "severity": bug.severity,
|
| |
+ "status": bug.status,
|
| |
+ "modified": str(bug.last_change_time),
|
| |
+ "reported": str(bug.creation_time),
|
| |
+ "release": bug.version,
|
| |
+ "keywords": bug.keywords,
|
| |
+ "url": "https://bugzilla.redhat.com/%s" % bug.id
|
| |
+ })
|
| |
+ return data
|
| |
+
|
| |
+ def get_overrides():
|
| |
+ """
|
| |
+ Returns list of all active buildroot overrides
|
| |
+ """
|
| |
+ bc = BodhiClient(username="oraculum",
|
| |
+ useragent="Fedora Easy Karma/GIT",
|
| |
+ retries=3)
|
| |
+
|
| |
+ query_args = {"expired": False,
|
| |
+ "rows_per_page": 50,
|
| |
+ }
|
| |
+
|
| |
+ overrides = []
|
| |
+ try:
|
| |
+ # since bodhi has a query limit but multiple pages, get ALL of the
|
| |
+ # updates before starting to process
|
| |
+ result = bc.list_overrides(**query_args)
|
| |
+ overrides.extend(result['overrides'])
|
| |
+ while result.page < result.pages:
|
| |
+ next_page = result['page'] + 1
|
| |
+ app.logger.debug("Fetching overrides page {} of {}".format(
|
| |
+ next_page, result['pages']))
|
| |
+ result = bc.list_overrides(page=next_page, **query_args)
|
| |
+ app.logger.debug("Queried Bodhi page %s" % next_page)
|
| |
+ overrides.extend(result['overrides'])
|
| |
+ # There is no clear indication which Exceptions bc.query() might
|
| |
+ # throw, therefore catch all (python-fedora-0.3.32.3-1.fc19)
|
| |
+ except Exception as e:
|
| |
+ app.logger.error("Error while querying Bodhi: {0}".format(e))
|
| |
+
|
| |
+ return overrides
|
| |
+
|
| |
+ def get_user_overrides(packages, raw_overrides):
|
| |
+ """
|
| |
+ Returns all active buildroot overrides for single user
|
| |
+ Formatted as dict of dicts:
|
| |
+ package_name: {
|
| |
+ "pretty_name": override["nvr"],
|
| |
+ "url": "https://bodhi.fedoraproject.org/overrides/" + override["nvr"],
|
| |
+ "submission_date": override["submission_date"],
|
| |
+ "expiration_date": override["expiration_date"],
|
| |
+ "release": release_from_nevra(override["nvr"])
|
| |
+ }
|
| |
+ """
|
| |
+ data = {}
|
| |
+ for package in packages:
|
| |
+ data[package] = []
|
| |
+ for override in raw_overrides:
|
| |
+ for package in packages:
|
| |
+ if name_in_nevra(package, override["nvr"]):
|
| |
+ data[package].append(process_override(override))
|
| |
+ return data
|
| |
+
|
| |
+ if __name__ == "__main__":
|
| |
+ import ipdb; ipdb.set_trace()
|
| |
frantisekz
commented 3 years ago
Provides api/v1/packager_dashboard/<user> endpoint
Missing:
- FTBFS and FTI reporting
Postponed for later inclusion:
- Modules and Flatpaks support in bodhi data source