From d54063d69b171d5544942bc7339f000288be4a88 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Feb 12 2019 21:00:20 +0000 Subject: PR#891: Web UI python3 changes Merges #891 https://pagure.io/koji/pull-request/891 Fixes: #890 https://pagure.io/koji/issue/890 [RFE] Web UI can go Py3! --- diff --git a/koji.spec b/koji.spec index b274853..14893cd 100644 --- a/koji.spec +++ b/koji.spec @@ -7,6 +7,13 @@ %bcond_with python3 %endif +# don't build py2 packages for py3-only systems +%if 0%{with python3} && (0%{?fedora} > 32 || 0%{?rhel} > 7) + %define with_python2 0 +%else + %define with_python2 1 +%endif + # Compatibility with RHEL. These macros have been added to EPEL but # not yet to RHEL proper. # https://bugzilla.redhat.com/show_bug.cgi?id=1307190 @@ -58,6 +65,7 @@ BuildRequires: pkgconfig Koji is a system for building and tracking RPMS. The base package contains shared libraries and the command-line interface. +%if 0%{with_python2} %package -n python2-%{name} Summary: Build system tools python library %{?python_provide:%python_provide python2-%{name}} @@ -79,6 +87,7 @@ Requires: python-six %description -n python2-%{name} desc +%endif %if 0%{with python3} %package -n python%{python3_pkgversion}-%{name} @@ -100,6 +109,7 @@ Requires: python%{python3_pkgversion}-six desc %endif +%if 0%{with_python2} %package -n python2-%{name}-cli-plugins Summary: Koji client plugins Group: Applications/Internet @@ -108,6 +118,7 @@ Requires: %{name} = %{version}-%{release} %description -n python2-%{name}-cli-plugins Plugins to the koji command-line interface +%endif %if 0%{with python3} %package -n python%{python3_pkgversion}-%{name}-cli-plugins @@ -307,10 +318,12 @@ Requires(postun): systemd %description utils Utilities for the Koji system -%package web +%if 0%{with_python2} +%package -n python2-%{name}-web Summary: Koji Web UI Group: Applications/Internet License: LGPLv2 +%{?python_provide:%python_provide python2-%{name}-web} Requires: httpd Requires: mod_wsgi %if 0%{?fedora} >= 21 || 0%{?rhel} >= 7 @@ -321,12 +334,32 @@ Requires: python-krbV >= 1.0.13 %endif Requires: python-psycopg2 Requires: python-cheetah -Requires: %{name} = %{version}-%{release} -# we need the python2 lib here Requires: python2-%{name} = %{version}-%{release} +Provides: koji-web = %{version}-%{release} +Obsoletes: koji-web < 1.16.2 -%description web +%description -n python2-%{name}-web koji-web is a web UI to the Koji system. +%endif + +%if 0%{with python3} +%package -n python%{python3_pkgversion}-%{name}-web +Summary: Koji Web UI +Group: Applications/Internet +License: LGPLv2 +%{?python_provide:%python_provide python%{python3_pkgversion}-%{name}-web} +Requires: httpd +Requires: python%{python3_pkgversion}-mod_wsgi +Requires: mod_auth_gssapi +Requires: python%{python3_pkgversion}-psycopg2 +Requires: python%{python3_pkgversion}-cheetah +Requires: python%{python3_pkgversion}-%{name} = %{version}-%{release} +Provides: koji-web = %{version}-%{release} +Obsoletes: koji-web < 1.16.2 + +%description -n python%{python3_pkgversion}-%{name}-web +koji-web is a web UI to the Koji system. +%endif %prep %setup -q @@ -335,9 +368,11 @@ koji-web is a web UI to the Koji system. %install rm -rf $RPM_BUILD_ROOT +%if 0%{with_python2} make DESTDIR=$RPM_BUILD_ROOT PYTHON=%{__python2} %{?install_opt} install +%endif %if 0%{with python3} -for d in koji cli plugins hub ; do +for d in koji cli plugins hub www ; do pushd $d make DESTDIR=$RPM_BUILD_ROOT PYTHON=%{__python3} %{?install_opt} install popd @@ -356,10 +391,12 @@ rm -rf $RPM_BUILD_ROOT %dir /etc/koji.conf.d %doc docs Authors COPYING LGPL +%if 0%{with_python2} %files -n python2-%{name} %defattr(-,root,root) %{python2_sitelib}/%{name} %{python2_sitelib}/koji_cli +%endif %if 0%{with python3} %files -n python%{python3_pkgversion}-koji @@ -367,12 +404,14 @@ rm -rf $RPM_BUILD_ROOT %{python3_sitelib}/koji_cli %endif +%if 0%{with_python2} %files -n python2-%{name}-cli-plugins %defattr(-,root,root) %{python2_sitelib}/koji_cli_plugins # we don't have config files for default plugins yet #%%dir %%{_sysconfdir}/koji/plugins #%%config(noreplace) %%{_sysconfdir}/koji/plugins/*.conf +%endif %if 0%{with python3} %files -n python%{python3_pkgversion}-%{name}-cli-plugins @@ -390,10 +429,12 @@ rm -rf $RPM_BUILD_ROOT %config(noreplace) /etc/koji-hub/hub.conf %dir /etc/koji-hub/hub.conf.d +%if 0%{with_python2} %files -n python2-%{name}-hub %defattr(-,root,root) %{_datadir}/koji-hub/*.py* %dir %{_libexecdir}/koji-hub +%endif %if 0%{with python3} %files -n python%{python3_pkgversion}-%{name}-hub @@ -407,9 +448,11 @@ rm -rf $RPM_BUILD_ROOT %dir /etc/koji-hub/plugins %config(noreplace) /etc/koji-hub/plugins/*.conf +%if 0%{with_python2} %files -n python2-%{name}-hub-plugins %defattr(-,root,root) %{_prefix}/lib/koji-hub-plugins/*.py* +%endif %if 0%{with python3} %files -n python%{python3_pkgversion}-%{name}-hub-plugins @@ -418,13 +461,16 @@ rm -rf $RPM_BUILD_ROOT %{_prefix}/lib/koji-hub-plugins/__pycache__ %endif +%if 0%{with_python2} %files builder-plugins %defattr(-,root,root) %dir /etc/kojid/plugins %config(noreplace) /etc/kojid/plugins/*.conf %dir %{_prefix}/lib/koji-builder-plugins %{_prefix}/lib/koji-builder-plugins/*.py* +%endif +%if 0%{with_python2} %files utils %defattr(-,root,root) %{_sbindir}/kojira @@ -442,15 +488,29 @@ rm -rf $RPM_BUILD_ROOT %{_sbindir}/koji-shadow %dir /etc/koji-shadow %config(noreplace) /etc/koji-shadow/koji-shadow.conf +%endif -%files web +%if 0%{with_python2} +%files -n python2-%{name}-web %defattr(-,root,root) %{_datadir}/koji-web %dir /etc/kojiweb %config(noreplace) /etc/kojiweb/web.conf %config(noreplace) /etc/httpd/conf.d/kojiweb.conf %dir /etc/kojiweb/web.conf.d +%endif +%if 0%{with python3} +%files -n python%{python3_pkgversion}-%{name}-web +%defattr(-,root,root) +%{_datadir}/koji-web +%dir /etc/kojiweb +%config(noreplace) /etc/kojiweb/web.conf +%config(noreplace) /etc/httpd/conf.d/kojiweb.conf +%dir /etc/kojiweb/web.conf.d +%endif + +%if 0%{with_python2} %files builder %defattr(-,root,root) %{_sbindir}/kojid @@ -491,7 +551,9 @@ if [ $1 = 0 ]; then /sbin/chkconfig --del kojid fi %endif +%endif +%if 0%{with_python2} %files vm %defattr(-,root,root) %{_sbindir}/kojivmd @@ -550,6 +612,7 @@ if [ $1 = 0 ]; then /sbin/chkconfig --del kojira fi %endif +%endif %changelog * Tue May 15 2018 Mike McLean - 1.16.0-1 diff --git a/www/conf/kojiweb.conf b/www/conf/kojiweb.conf index 020fbc5..5c98c9c 100644 --- a/www/conf/kojiweb.conf +++ b/www/conf/kojiweb.conf @@ -2,6 +2,11 @@ Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py" #(configuration goes in /etc/kojiweb/web.conf) +# Python 3 Cheetah expectes unicode everywhere, apache's default lang is C +# which is not sufficient to open our templates +WSGIDaemonProcess koji lang=C.UTF-8 +WSGIProcessGroup koji + Options ExecCGI SetHandler wsgi-script diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index cfff038..cd9844d 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -129,8 +129,7 @@ #end for #end if #set $arches = $rpmsByArch.keys() - #silent $arches.sort() - #for $arch in $arches + #for $arch in sorted($arches) #if $arch == 'src' #silent continue #end if diff --git a/www/kojiweb/error.chtml b/www/kojiweb/error.chtml index 5113805..9d48a32 100644 --- a/www/kojiweb/error.chtml +++ b/www/kojiweb/error.chtml @@ -1,4 +1,3 @@ - #from kojiweb import util #include "includes/header.chtml" diff --git a/www/kojiweb/includes/header.chtml b/www/kojiweb/includes/header.chtml index 6afb797..a26e818 100644 --- a/www/kojiweb/includes/header.chtml +++ b/www/kojiweb/includes/header.chtml @@ -1,4 +1,4 @@ -#encoding utf-8 +#encoding UTF-8 #import koji #from kojiweb import util #import random diff --git a/www/kojiweb/index.chtml b/www/kojiweb/index.chtml index 738256c..2961bd4 100644 --- a/www/kojiweb/index.chtml +++ b/www/kojiweb/index.chtml @@ -4,7 +4,7 @@ #include "includes/header.chtml"
$welcomeMessage
- +
#if $user then 'Your ' else ''#Recent Builds
@@ -36,7 +36,7 @@

- +
#if $user then 'Your ' else ''#Recent Tasks
diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 39c6188..5f0df31 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -43,7 +43,7 @@ from six.moves import range import six # Convenience definition of a commonly-used sort function -_sortbyname = kojiweb.util.sortByKeyFunc('name') +_sortbyname = lambda x: x['name'] #loggers authlogger = logging.getLogger('koji.auth') @@ -304,7 +304,7 @@ def index(environ, packageOrder='package_name', packageStart=None): start=packageStart, dataName='packages', prefix='package', order=packageOrder, pageSize=10) notifs = server.getBuildNotifications(user['id']) - notifs.sort(kojiweb.util.sortByKeyFunc('id')) + notifs.sort(key=lambda x: x['id']) # XXX Make this a multicall for notif in notifs: notif['package'] = None @@ -898,9 +898,9 @@ def taginfo(environ, tagID, all='0', packageOrder='package_name', packageStart=N tagsByChild[child_id].append(child_id) srcTargets = server.getBuildTargets(buildTagID=tag['id']) - srcTargets.sort(_sortbyname) + srcTargets.sort(key=_sortbyname) destTargets = server.getBuildTargets(destTagID=tag['id']) - destTargets.sort(_sortbyname) + destTargets.sort(key=_sortbyname) values['tag'] = tag values['tagID'] = tag['id'] @@ -1108,9 +1108,9 @@ def buildinfo(environ, buildID): values['title'] = koji.buildLabel(build) + ' | Build Info' tags = server.listTags(build['id']) - tags.sort(_sortbyname) + tags.sort(key=_sortbyname) rpms = server.listBuildRPMs(build['id']) - rpms.sort(_sortbyname) + rpms.sort(key=_sortbyname) typeinfo = server.getBuildType(buildID) archiveIndex = {} for btype in typeinfo: @@ -1365,23 +1365,23 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i builtInRoot = server.getBuildroot(rpm['buildroot_id']) if rpm['external_repo_id'] == 0: values['provides'] = server.getRPMDeps(rpm['id'], koji.DEP_PROVIDE) - values['provides'].sort(_sortbyname) + values['provides'].sort(key=_sortbyname) values['obsoletes'] = server.getRPMDeps(rpm['id'], koji.DEP_OBSOLETE) - values['obsoletes'].sort(_sortbyname) + values['obsoletes'].sort(key=_sortbyname) values['conflicts'] = server.getRPMDeps(rpm['id'], koji.DEP_CONFLICT) - values['conflicts'].sort(_sortbyname) + values['conflicts'].sort(key=_sortbyname) values['requires'] = server.getRPMDeps(rpm['id'], koji.DEP_REQUIRE) - values['requires'].sort(_sortbyname) + values['requires'].sort(key=_sortbyname) if koji.RPM_SUPPORTS_OPTIONAL_DEPS: values['optional_deps'] = True values['recommends'] = server.getRPMDeps(rpm['id'], koji.DEP_RECOMMEND) - values['recommends'].sort(_sortbyname) + values['recommends'].sort(key=_sortbyname) values['suggests'] = server.getRPMDeps(rpm['id'], koji.DEP_SUGGEST) - values['suggests'].sort(_sortbyname) + values['suggests'].sort(key=_sortbyname) values['supplements'] = server.getRPMDeps(rpm['id'], koji.DEP_SUPPLEMENT) - values['supplements'].sort(_sortbyname) + values['supplements'].sort(key=_sortbyname) values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE) - values['enhances'].sort(_sortbyname) + values['enhances'].sort(key=_sortbyname) else: values['optional_deps'] = False headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license']) @@ -1544,10 +1544,10 @@ def hostinfo(environ, hostID=None, userID=None): values['title'] = host['name'] + ' | Host Info' channels = server.listChannels(host['id']) - channels.sort(_sortbyname) + channels.sort(key=_sortbyname) buildroots = server.listBuildroots(hostID=host['id'], state=[state[1] for state in koji.BR_STATES.items() if state[0] != 'EXPIRED']) - buildroots.sort(kojiweb.util.sortByKeyFunc('-create_event_time')) + buildroots.sort(key=lambda x: x['create_event_time'], reverse=True) values['host'] = host values['channels'] = channels @@ -1603,7 +1603,7 @@ def hostedit(environ, hostID): values['host'] = host allChannels = server.listChannels() - allChannels.sort(_sortbyname) + allChannels.sort(key=_sortbyname) values['allChannels'] = allChannels values['hostChannels'] = server.listChannels(hostID=host['id']) @@ -1646,7 +1646,7 @@ def channelinfo(environ, channelID): queryOpts={'countOnly': True}) hosts = server.listHosts(channelID=channelID) - hosts.sort(_sortbyname) + hosts.sort(key=_sortbyname) values['channel'] = channel values['hosts'] = hosts @@ -1843,7 +1843,7 @@ def buildtargetedit(environ, targetID): else: values = _initValues(environ, 'Edit Build Target', 'buildtargets') tags = server.listTags() - tags.sort(_sortbyname) + tags.sort(key=_sortbyname) values['target'] = target values['tags'] = tags @@ -1877,7 +1877,7 @@ def buildtargetcreate(environ): values = _initValues(environ, 'Add Build Target', 'builtargets') tags = server.listTags() - tags.sort(_sortbyname) + tags.sort(key=_sortbyname) values['target'] = None values['tags'] = tags @@ -2163,7 +2163,7 @@ def recentbuilds(environ, user=None, tag=None, package=None): if tagObj != None: builds = server.listTagged(tagObj['id'], inherit=True, package=(packageObj and packageObj['name'] or None), owner=(userObj and userObj['name'] or None)) - builds.sort(kojiweb.util.sortByKeyFunc('-completion_time', noneGreatest=True)) + builds.sort(key=kojiweb.util.sortByKeyFuncNoneGreatest('completion_time'), reverse=True) builds = builds[:20] else: kwargs = {} diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 5ee634a..69c2bbd 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -1,6 +1,6 @@ #import koji #from kojiweb import util -#import urllib +#from six.moves.urllib.parse import quote #import cgi #def printValue($key, $value, $sep=', ') @@ -424,9 +424,9 @@ $value
Output #for $volume, $filename in $output - $filename + $filename #if $filename.endswith('.log') - (tail) + (tail) #end if
#end for diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index df17a1d..abb3424 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -184,7 +184,7 @@ class Dispatcher(object): name = 'koji' + name elif not name.startswith('koji'): name = 'koji.' + name - level_code = logging._levelNames[level] + level_code = logging.getLevelName(level) logging.getLogger(name).setLevel(level_code) logger = logging.getLogger("koji") # if KojiDebug is set, force main log level to DEBUG @@ -350,7 +350,15 @@ class Dispatcher(object): ('Content-Length', str(len(result))), ('Content-Type', 'text/html'), ] - return [result], headers + return self._tobytes(result), headers + + def _tobytes(self, result): + if isinstance(result, six.string_types): + if six.PY2: + result = [result] + else: + result = [bytes(result, encoding='utf-8')] + return result def handle_request(self, environ, start_response): if self.startup_error: @@ -405,9 +413,7 @@ class Dispatcher(object): self.logger.debug("Headers:") self.logger.debug(koji.util.LazyString(pprint.pformat, [headers])) start_response(status, headers) - if isinstance(result, six.string_types): - result = [result] - return result + return self._tobytes(result) def application(self, environ, start_response): """wsgi handler""" diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index c32f7f9..ed4571f 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -19,22 +19,24 @@ # Authors: # Mike Bonnet # Mike McLean - from __future__ import absolute_import from __future__ import division +import cgi import Cheetah.Template import datetime import koji from koji.util import md5_constructor import os +import six +import ssl import stat + +from six.moves import range #a bunch of exception classes that explainError needs from socket import error as socket_error -from socket import sslerror as socket_sslerror from six.moves.xmlrpc_client import ProtocolError from xml.parsers.expat import ExpatError -import cgi -from six.moves import range + class NoSuchException(Exception): pass @@ -97,7 +99,7 @@ class DecodeUTF8(Cheetah.Filters.Filter): def filter(self, *args, **kw): """Convert all strs to unicode objects""" result = super(DecodeUTF8, self).filter(*args, **kw) - if isinstance(result, unicode): + if isinstance(result, six.text_type): pass else: result = result.decode('utf-8', 'replace') @@ -147,7 +149,10 @@ def _genHTML(environ, fileName): tmpl_class = Cheetah.Template.Template.compile(file=fileName) TEMPLATES[fileName] = tmpl_class tmpl_inst = tmpl_class(namespaces=[environ['koji.values']], filter=XHTMLFilter) - return tmpl_inst.respond().encode('utf-8', 'replace') + if six.PY2: + return tmpl_inst.respond().encode('utf-8', 'replace') + else: + return tmpl_inst.respond() def _truncTime(): now = datetime.datetime.now() @@ -243,25 +248,15 @@ def passthrough_except(template, *exclude): passvars.append(var) return passthrough(template, *passvars) -def sortByKeyFunc(key, noneGreatest=False): +def sortByKeyFuncNoneGreatest(key): """Return a function to sort a list of maps by the given key. - If the key starts with '-', sort in reverse order. If noneGreatest - is True, None will sort higher than all other values (instead of lower). + None will sort higher than all other values (instead of lower). """ - if noneGreatest: - # Normally None evaluates to be less than every other value - # Invert the comparison so it always evaluates to greater - cmpFunc = lambda a, b: (a is None or b is None) and -(cmp(a, b)) or cmp(a, b) - else: - cmpFunc = cmp - - if key.startswith('-'): - key = key[1:] - sortFunc = lambda a, b: cmpFunc(b[key], a[key]) - else: - sortFunc = lambda a, b: cmpFunc(a[key], b[key]) - - return sortFunc + def internal_key(obj): + v = obj[key] + # Nones has priority, others are second + return (v is None, v) + return internal_key def paginateList(values, data, start, dataName, prefix=None, order=None, noneGreatest=False, pageSize=50): """ @@ -273,7 +268,12 @@ def paginateList(values, data, start, dataName, prefix=None, order=None, noneGre be added to the value map. """ if order != None: - data.sort(sortByKeyFunc(order, noneGreatest)) + if order.startswith('-'): + order = order[1:] + reverse = True + else: + reverse = False + data.sort(key=sortByKeyFuncNoneGreatest(order), reverse=reverse) totalRows = len(data) @@ -359,7 +359,7 @@ def _populateValues(values, dataName, prefix, data, totalRows, start, count, pag values[(prefix and prefix + 'Order' or 'order')] = order currentPage = start // pageSize values[(prefix and prefix + 'CurrentPage' or 'currentPage')] = currentPage - totalPages = totalRows // pageSize + totalPages = int(totalRows // pageSize) if totalRows % pageSize > 0: totalPages += 1 pages = [page for page in range(0, totalPages) if (abs(page - currentPage) < 100 or ((page + 1) % 100 == 0))] @@ -531,6 +531,9 @@ def escapeHTML(value): return value value = koji.fixEncoding(value) + if six.PY3: + # it is bytes now, so decode to str + value = value.decode() return value.replace('&', '&').\ replace('<', '<').\ replace('>', '>') @@ -594,7 +597,7 @@ bug, a server configuration issue, or possibly something else.""" str = """\ An error has occurred in the web interface code. This could be due to \ a bug or a configuration issue.""" - elif isinstance(error, (socket_error, socket_sslerror)): + elif isinstance(error, (socket_error, ssl.SSLError)): str = """\ The web interface is having difficulty communicating with the main \ server. This most likely indicates a network issue.""" @@ -611,7 +614,7 @@ a network issue or load issues on the server.""" class TaskResultFragment(object): """Represent an HTML fragment composed from texts and tags. - + This class permits us to compose HTML fragment by the default composer method or self-defined composer function. @@ -644,7 +647,6 @@ class TaskResultFragment(object): self.empty_str_placeholder = empty_str_placeholder def default_composer(self, length=None): - import cgi if length is None: text = self.text else: @@ -691,7 +693,6 @@ class TaskResultLine(object): self.size=self._size() def default_composer(self, length=None, postscript=None): - import cgi line_text = '' size = 0 if postscript is None: