From d37cc1b522b4713d17ad2989f71717ea86479aa3 Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Jun 05 2018 07:15:02 +0000 Subject: [PATCH 1/3] Jenkinsfile: tidy up package installation for functional tests --- diff --git a/Jenkinsfile b/Jenkinsfile index b469667..2a49b8c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -123,14 +123,12 @@ node('docker') { node('fedora') { checkout scm - sh ''' - sudo dnf -y builddep greenwave.spec - sudo dnf -y install python2-flake8 python2-pylint python2-sphinx python-sphinxcontrib-httpdomain - ''' + /* Install packages needed by the functional tests. */ + sh 'sudo dnf -y install python2-pytest python2-requests python2-sqlalchemy python2-gunicorn' - sh ''' - sudo dnf -y install /usr/bin/py.test python-sqlalchemy python-resultsdb_api python-gunicorn python2-mock - ''' + /* Also need to install Greenwave's dependencies, since we are running it + * locally not in Openshift for now. */ + sh 'sudo dnf -y builddep greenwave.spec' def openshiftHost = 'greenwave-test.cloud.upshift.engineering.redhat.com' def waiverdbURL = "waiverdb-test-${env.BUILD_TAG}-web-${openshiftHost}" From 6d877ef089ff5d094f4bb13cd0126308a04f3ff0 Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Jun 06 2018 04:02:06 +0000 Subject: [PATCH 2/3] switch to Python 3 --- diff --git a/Dockerfile b/Dockerfile index 901883f..babaf0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,8 @@ ARG cacert_url=undefined COPY $greenwave_rpm /tmp RUN dnf -y install \ - python2-gunicorn \ - python-memcached \ + python3-gunicorn \ + python3-memcached \ /tmp/$(basename $greenwave_rpm) \ && dnf -y clean all \ && rm -rf /tmp/* @@ -25,4 +25,4 @@ RUN if [ "$cacert_url" != "undefined" ]; then \ fi USER 1001 EXPOSE 8080 -ENTRYPOINT gunicorn --workers 8 --bind 0.0.0.0:8080 --access-logfile=- --enable-stdio-inheritance greenwave.wsgi:app +ENTRYPOINT gunicorn-3 --workers 8 --bind 0.0.0.0:8080 --access-logfile=- --enable-stdio-inheritance greenwave.wsgi:app diff --git a/Jenkinsfile b/Jenkinsfile index 2a49b8c..e7c7a9b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,14 +8,14 @@ timestamps { node('fedora') { checkout scm sh 'sudo dnf -y builddep greenwave.spec' - sh 'sudo dnf -y install python2-flake8 python2-pylint python2-sphinx python-sphinxcontrib-httpdomain' + sh 'sudo dnf -y install python3-flake8 python3-pylint python3-sphinx python3-sphinxcontrib-httpdomain' /* Needed to get the latest /etc/mock/fedora-28-x86_64.cfg */ sh 'sudo dnf -y update mock-core-configs' stage('Invoke Flake8') { - sh 'flake8' + sh 'flake8-3' } stage('Invoke Pylint') { - sh 'pylint-2 --reports=n greenwave' + sh 'pylint-3 --reports=n greenwave' } stage('Build Docs') { sh 'DEV=true GREENWAVE_CONFIG=$(pwd)/conf/settings.py.example make -C docs html' @@ -91,7 +91,7 @@ node('docker') { def f28_rpm = findFiles(glob: 'mock-result/f28/**/*.noarch.rpm')[0] def appversion = sh(returnStdout: true, script: """ rpm2cpio ${f28_rpm} | \ - cpio --quiet --extract --to-stdout ./usr/lib/python2.7/site-packages/greenwave\\*.egg-info/PKG-INFO | \ + cpio --quiet --extract --to-stdout ./usr/lib/python3\\*/site-packages/greenwave\\*.egg-info/PKG-INFO | \ awk '/^Version: / {print \$2}' """).trim() /* Git builds will have a version like 0.3.2.dev1+git.3abbb08 following @@ -124,7 +124,7 @@ node('fedora') { checkout scm /* Install packages needed by the functional tests. */ - sh 'sudo dnf -y install python2-pytest python2-requests python2-sqlalchemy python2-gunicorn' + sh 'sudo dnf -y install python3-pytest python3-requests python3-sqlalchemy python3-gunicorn' /* Also need to install Greenwave's dependencies, since we are running it * locally not in Openshift for now. */ @@ -200,7 +200,7 @@ node('fedora') { ,"REQUESTS_CA_BUNDLE=${env.WORKSPACE}/ca-chain.crt" ,"WAIVERDB_TEST_URL=https://${waiverdbURL}/" ,"RESULTSDB_TEST_URL=https://${resultsdbURL}/"]) { - sh 'py.test -v --junitxml=junit-functional-tests.xml functional-tests/' + sh 'py.test-3 -v --junitxml=junit-functional-tests.xml functional-tests/' } junit 'junit-functional-tests.xml' } finally { diff --git a/docs/Makefile b/docs/Makefile index db4017e..79ffc33 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = -W -SPHINXBUILD = sphinx-build +SPHINXBUILD = sphinx-build-3 SPHINXPROJ = Greenwave SOURCEDIR = . BUILDDIR = _build diff --git a/docs/conf.py b/docs/conf.py index 237316c..ed12775 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -183,7 +183,7 @@ epub_exclude_files = ['search.html'] # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/2/': 'python-intersphinx.inv'} +intersphinx_mapping = {'https://docs.python.org/3/': 'python-intersphinx.inv'} # Configuration for issue tracker magic linking. issuetracker = None diff --git a/docs/dev-guide.rst b/docs/dev-guide.rst index f60c478..b900335 100644 --- a/docs/dev-guide.rst +++ b/docs/dev-guide.rst @@ -39,7 +39,7 @@ the following command: .. code-block:: console - $ py.test greenwave/tests/ + $ py.test-3 greenwave/tests/ There are also functional tests in the :file:`functional-tests` directory. The functional tests will start their own copy of the `ResultsDB`_, `WaiverDB`_, @@ -48,7 +48,7 @@ functional tests like this: .. code-block:: console - $ PYTHONPATH=. py.test functional-tests/ + $ PYTHONPATH=. py.test-3 functional-tests/ The functional tests assume you have ResultsDB and WaiverDB git checkouts in :file:`../resultsdb` and :file:`../waiverdb` respectively. You can tell it to diff --git a/functional-tests/conftest.py b/functional-tests/conftest.py index d468da8..48679da 100644 --- a/functional-tests/conftest.py +++ b/functional-tests/conftest.py @@ -139,7 +139,7 @@ def bodhi(): yield os.environ['BODHI_TEST_URL'] else: # Start fake Bodhi as a subprocess - p = subprocess.Popen(['gunicorn', + p = subprocess.Popen(['gunicorn-3', '--bind=127.0.0.1:5677', '--access-logfile=-', '--pythonpath=' + os.path.dirname(__file__), @@ -158,7 +158,7 @@ def distgit_server(tmpdir_factory): tmp_dir = tmpdir_factory.mktemp('distgit') f = open(tmp_dir.strpath + "/gating.yaml", "w+") f.close() - p = subprocess.Popen([sys.executable, '-m', 'SimpleHTTPServer', '5678'], cwd=tmp_dir.strpath) + p = subprocess.Popen([sys.executable, '-m', 'http.server', '5678'], cwd=tmp_dir.strpath) log.debug('Started dist-git server as pid %s', p.pid) wait_for_listen(5678) yield 'http://localhost:5678' @@ -194,7 +194,7 @@ def greenwave_server(tmpdir_factory, resultsdb_server, waiverdb_server, bodhi): env = dict(os.environ, PYTHONPATH='.', GREENWAVE_CONFIG=settings_file.strpath) - p = subprocess.Popen(['gunicorn', + p = subprocess.Popen(['gunicorn-3', '--bind=127.0.0.1:5005', '--access-logfile=-', 'greenwave.wsgi:app'], @@ -229,10 +229,10 @@ class TestDataBuilder(object): self._counter = itertools.count(1) def unique_nvr(self, name='glibc'): - return '{}-1.0-{}.el7'.format(name, self._counter.next()) + return '{}-1.0-{}.el7'.format(name, next(self._counter)) def unique_compose_id(self): - return 'Fedora-9000-19700101.n.{}'.format(self._counter.next()) + return 'Fedora-9000-19700101.n.{}'.format(next(self._counter)) def _create_result(self, data): response = self.requests_session.post( diff --git a/functional-tests/consumers/test_resultsdb.py b/functional-tests/consumers/test_resultsdb.py index 1e39956..12207c2 100644 --- a/functional-tests/consumers/test_resultsdb.py +++ b/functional-tests/consumers/test_resultsdb.py @@ -82,7 +82,7 @@ def test_consume_new_result( 'applicable_policies': ['taskotron_release_critical_tasks_with_blacklist', 'taskotron_release_critical_tasks'], 'policies_satisfied': False, - 'summary': u'3 of 3 required test results missing', + 'summary': '3 of 3 required test results missing', 'unsatisfied_requirements': [ { 'testcase': 'dist.abicheck', @@ -175,7 +175,7 @@ def test_consume_new_result( 'applicable_policies': ['taskotron_release_critical_tasks_with_blacklist', 'taskotron_release_critical_tasks'], 'policies_satisfied': False, - 'summary': u'3 of 3 required test results missing', + 'summary': '3 of 3 required test results missing', 'unsatisfied_requirements': [ { 'testcase': 'dist.abicheck', @@ -364,16 +364,16 @@ def test_invalidate_new_result_with_real_cache( # Now, handle a message about the new failing result message = { 'body': { - u'topic': u'resultsdb.result.new', - u'msg': { - u'id': u'whatever', - u'outcome': u'doesn\'t matter', - u'testcase': { - u'name': u'dist.rpmdeplint' + 'topic': 'resultsdb.result.new', + 'msg': { + 'id': 'whatever', + 'outcome': 'doesn\'t matter', + 'testcase': { + 'name': 'dist.rpmdeplint' }, - u'data': { - u'item': nvr.decode('utf-8'), - u'type': u'koji_build', + 'data': { + 'item': nvr, + 'type': 'koji_build', } } } @@ -494,22 +494,22 @@ def test_consume_compose_id_result( assert r.status_code == 200 old_decision = r.json() msg = { - u'applicable_policies': [u'openqa_important_stuff_for_rawhide'], - u'decision_context': u'rawhide_compose_sync_to_mirrors', - u'policies_satisfied': False, + 'applicable_policies': ['openqa_important_stuff_for_rawhide'], + 'decision_context': 'rawhide_compose_sync_to_mirrors', + 'policies_satisfied': False, 'product_version': 'fedora-rawhide', - 'subject': [{u'productmd.compose.id': compose_id}], + 'subject': [{'productmd.compose.id': compose_id}], 'subject_type': 'compose', 'subject_identifier': compose_id, - u'summary': u'1 of 2 required test results missing', + 'summary': '1 of 2 required test results missing', 'previous': old_decision, - u'unsatisfied_requirements': [{ - u'item': {u'productmd.compose.id': compose_id}, + 'unsatisfied_requirements': [{ + 'item': {'productmd.compose.id': compose_id}, 'subject_type': 'compose', 'subject_identifier': compose_id, - u'scenario': u'scenario2', - u'testcase': u'compose.install_no_user', - u'type': u'test-result-missing'} + 'scenario': 'scenario2', + 'testcase': 'compose.install_no_user', + 'type': 'test-result-missing'} ] } diff --git a/functional-tests/consumers/test_waiverdb.py b/functional-tests/consumers/test_waiverdb.py index 7fe98bd..77a0f54 100644 --- a/functional-tests/consumers/test_waiverdb.py +++ b/functional-tests/consumers/test_waiverdb.py @@ -58,7 +58,7 @@ def test_consume_new_waiver( 'applicable_policies': ['taskotron_release_critical_tasks_with_blacklist', 'taskotron_release_critical_tasks'], 'policies_satisfied': False, - 'summary': u'1 of 3 required tests failed', + 'summary': '1 of 3 required tests failed', 'unsatisfied_requirements': [ { 'result_id': result['id'], @@ -88,7 +88,7 @@ def test_consume_new_waiver( 'applicable_policies': ['taskotron_release_critical_tasks_with_blacklist', 'taskotron_release_critical_tasks'], 'policies_satisfied': False, - 'summary': u'1 of 3 required tests failed', + 'summary': '1 of 3 required tests failed', 'unsatisfied_requirements': [ { 'result_id': result['id'], diff --git a/functional-tests/fake_bodhi.py b/functional-tests/fake_bodhi.py index bf2ec29..b5dfabc 100644 --- a/functional-tests/fake_bodhi.py +++ b/functional-tests/fake_bodhi.py @@ -2,7 +2,7 @@ import re import json import hashlib -from urlparse import parse_qs +from urllib.parse import parse_qs updates = {} #: {id -> update info} @@ -17,7 +17,7 @@ def application(environ, start_response): if environ['REQUEST_METHOD'] == 'GET': if updateid in updates: start_response('200 OK', [('Content-Type', 'application/json')]) - return [json.dumps({'update': updates[updateid]})] + return [json.dumps({'update': updates[updateid]}).encode('utf8')] else: start_response('404 Not Found', []) return [] @@ -34,16 +34,16 @@ def application(environ, start_response): if set(params['builds']).issubset(build['nvr'] for build in u['builds'])] else: - response_updates = updates.values() + response_updates = list(updates.values()) response_data = { 'page': 1, 'pages': 1, - 'rows_per_page': len(updates.values()), - 'total': len(updates.values()), + 'rows_per_page': len(updates), + 'total': len(updates), 'updates': response_updates, } start_response('200 OK', [('Content-Type', 'application/json')]) - return [json.dumps(response_data)] + return [json.dumps(response_data).encode('utf8')] if environ['REQUEST_METHOD'] == 'POST': body = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH'])) updateid = 'FEDORA-2000-{}'.format(hashlib.sha1(body).hexdigest()[-8:]) @@ -53,7 +53,7 @@ def application(environ, start_response): updates[updateid] = update print('Fake Bodhi created new update %r' % update) start_response('201 Created', [('Content-Type', 'application/json')]) - return [json.dumps(update)] # XXX check what Bodhi really returns + return [json.dumps(update).encode('utf8')] # XXX check what Bodhi really returns else: start_response('405 Method Not Allowed', []) return [] diff --git a/functional-tests/test_api_v1.py b/functional-tests/test_api_v1.py index dcf29ff..fcf62cf 100644 --- a/functional-tests/test_api_v1.py +++ b/functional-tests/test_api_v1.py @@ -73,7 +73,7 @@ def test_cannot_make_decision_without_product_version(requests_session, greenwav headers={'Content-Type': 'application/json'}, data=json.dumps(data)) assert r.status_code == 400 - assert u'Missing required product version' == r.json()['message'] + assert 'Missing required product version' == r.json()['message'] def test_cannot_make_decision_without_decision_context(requests_session, greenwave_server): @@ -86,7 +86,7 @@ def test_cannot_make_decision_without_decision_context(requests_session, greenwa headers={'Content-Type': 'application/json'}, data=json.dumps(data)) assert r.status_code == 400 - assert u'Missing required decision context' == r.json()['message'] + assert 'Missing required decision context' == r.json()['message'] def test_cannot_make_decision_without_subject_type(requests_session, greenwave_server): @@ -99,7 +99,7 @@ def test_cannot_make_decision_without_subject_type(requests_session, greenwave_s headers={'Content-Type': 'application/json'}, data=json.dumps(data)) assert r.status_code == 400 - assert u'Missing required "subject_type" parameter' == r.json()['message'] + assert 'Missing required "subject_type" parameter' == r.json()['message'] def test_cannot_make_decision_without_subject_identifier(requests_session, greenwave_server): @@ -112,7 +112,7 @@ def test_cannot_make_decision_without_subject_identifier(requests_session, green headers={'Content-Type': 'application/json'}, data=json.dumps(data)) assert r.status_code == 400 - assert u'Missing required "subject_identifier" parameter' == r.json()['message'] + assert 'Missing required "subject_identifier" parameter' == r.json()['message'] def test_cannot_make_decision_with_invalid_subject(requests_session, greenwave_server): @@ -151,8 +151,8 @@ def test_404_for_invalid_product_version(requests_session, greenwave_server, tes headers={'Content-Type': 'application/json'}, data=json.dumps(data)) assert r.status_code == 404 - expected = (u'Cannot find any applicable policies for bodhi_update subjects ' - u'at gating point bodhi_push_update_stable in f26') + expected = ('Cannot find any applicable policies for bodhi_update subjects ' + 'at gating point bodhi_push_update_stable in f26') assert expected == r.json()['message'] @@ -168,8 +168,8 @@ def test_404_for_invalid_decision_context(requests_session, greenwave_server, te headers={'Content-Type': 'application/json'}, data=json.dumps(data)) assert r.status_code == 404 - expected = (u'Cannot find any applicable policies for bodhi_update subjects ' - u'at gating point bodhi_push_update in fedora-26') + expected = ('Cannot find any applicable policies for bodhi_update subjects ' + 'at gating point bodhi_push_update in fedora-26') assert expected == r.json()['message'] @@ -320,7 +320,7 @@ def test_make_a_decision_on_failed_result(requests_session, greenwave_server, te 'scenario': None, } for name in TASKTRON_RELEASE_CRITICAL_TASKS[1:] ] - assert sorted(res_data['unsatisfied_requirements']) == sorted(expected_unsatisfied_requirements) + assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements def test_make_a_decision_on_no_results(requests_session, greenwave_server, testdatabuilder): @@ -567,11 +567,11 @@ def test_make_a_decision_on_failing_result_with_scenario( expected_summary = '1 of 2 required tests failed' assert res_data['summary'] == expected_summary expected_unsatisfied_requirements = [{ - u'item': {u'productmd.compose.id': compose_id}, - u'result_id': result['id'], - u'testcase': testcase_name, - u'type': u'test-result-failed', - u'scenario': u'scenario2', + 'item': {'productmd.compose.id': compose_id}, + 'result_id': result['id'], + 'testcase': testcase_name, + 'type': 'test-result-failed', + 'scenario': 'scenario2', }] assert res_data['unsatisfied_requirements'] == expected_unsatisfied_requirements diff --git a/greenwave.spec b/greenwave.spec index 0c1db63..01bd38b 100644 --- a/greenwave.spec +++ b/greenwave.spec @@ -9,51 +9,23 @@ License: GPLv2+ URL: https://pagure.io/greenwave Source0: https://files.pythonhosted.org/packages/source/g/%{name}/%{name}-%{upstream_version}.tar.gz -BuildRequires: python2-devel -%if 0%{?fedora} || 0%{?rhel} > 7 -BuildRequires: python2-setuptools -BuildRequires: python2-sphinx -BuildRequires: python2-sphinxcontrib-httpdomain -%if 0%{?fedora} >= 27 -BuildRequires: python2-sphinxcontrib-issuetracker -%else # old name -BuildRequires: python-sphinxcontrib-issuetracker -%endif -BuildRequires: python2-flask -BuildRequires: python2-pytest -BuildRequires: python2-requests -%if 0%{?fedora} >= 28 -BuildRequires: python2-pyyaml -%else # old name -BuildRequires: PyYAML -%endif -BuildRequires: python2-dogpile-cache -%else # EPEL7 uses python- naming -BuildRequires: python-setuptools -BuildRequires: python-flask -BuildRequires: pytest -BuildRequires: python-requests -BuildRequires: PyYAML -BuildRequires: python-dogpile-cache -%endif -BuildRequires: fedmsg +BuildRequires: python3-devel +BuildRequires: python3-setuptools +BuildRequires: python3-sphinx +BuildRequires: python3-sphinxcontrib-httpdomain +BuildRequires: python3-sphinxcontrib-issuetracker +BuildRequires: python3-flask +BuildRequires: python3-pytest +BuildRequires: python3-requests +BuildRequires: python3-PyYAML +BuildRequires: python3-dogpile-cache +BuildRequires: python3-fedmsg BuildArch: noarch -%if 0%{?fedora} || 0%{?rhel} > 7 -Requires: python2-flask -Requires: python2-requests -%if 0%{?fedora} >= 28 -Requires: python2-pyyaml -%else # old name -Requires: PyYAML -%endif -Requires: python2-dogpile-cache -%else # EPEL7 uses python- naming -Requires: python-flask -Requires: python-requests -Requires: PyYAML -Requires: python-dogpile-cache -%endif -Requires: fedmsg +Requires: python3-flask +Requires: python3-requests +Requires: python3-PyYAML +Requires: python3-dogpile-cache +Requires: python3-fedmsg %description Greenwave is a service for gating on automated tests by querying ResultsDB and @@ -63,26 +35,22 @@ WaiverDB. %setup -q -n %{name}-%{upstream_version} %build -%py2_build -%if 0%{?fedora} +%py3_build DEV=true GREENWAVE_CONFIG=$(pwd)/conf/settings.py.example make -C docs SPHINXOPTS= html text -%endif %install -%py2_install +%py3_install %check -export PYTHONPATH=%{buildroot}/%{python2_sitelib} -py.test greenwave/tests/ +export PYTHONPATH=%{buildroot}/%{python3_sitelib} +py.test-3 greenwave/tests/ %files %license COPYING %doc README.md conf -%if 0%{?fedora} %doc docs/_build/html docs/_build/text -%endif -%{python2_sitelib}/%{name} -%{python2_sitelib}/%{name}*.egg-info +%{python3_sitelib}/%{name} +%{python3_sitelib}/%{name}*.egg-info %{_sysconfdir}/fedmsg.d/* %changelog diff --git a/greenwave/app_factory.py b/greenwave/app_factory.py index d9c2f0b..e5c9084 100644 --- a/greenwave/app_factory.py +++ b/greenwave/app_factory.py @@ -2,10 +2,9 @@ from flask import Flask from greenwave.api_v1 import api -from greenwave.utils import json_error, load_config +from greenwave.utils import json_error, load_config, sha1_mangle_key from dogpile.cache import make_region -from dogpile.cache.util import sha1_mangle_key from requests import ConnectionError, Timeout from werkzeug.exceptions import default_exceptions diff --git a/greenwave/consumers/resultsdb.py b/greenwave/consumers/resultsdb.py index 29e4767..6cd61b1 100644 --- a/greenwave/consumers/resultsdb.py +++ b/greenwave/consumers/resultsdb.py @@ -78,12 +78,12 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer): """ Decode either a string or a list of strings. """ if len(value) == 1: value = value[0] - return value.decode('utf-8') + return value if data.get('type') == 'bodhi_update' and 'item' in data: - yield (u'bodhi_update', _decode(data['item'])) + yield ('bodhi_update', _decode(data['item'])) if 'productmd.compose.id' in data: - yield (u'compose', _decode(data['productmd.compose.id'])) + yield ('compose', _decode(data['productmd.compose.id'])) if (data.get('type') == 'koji_build' and 'item' in data or data.get('type') == 'brew-build' and 'item' in data or 'original_spec_nvr' in data): @@ -91,13 +91,13 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer): nvr = _decode(data['item']) else: nvr = _decode(data['original_spec_nvr']) - yield (u'koji_build', nvr) + yield ('koji_build', nvr) # If the result is for a build, it may also influence the decision # about any update which the build is part of. if current_app.config['BODHI_URL']: updateid = greenwave.resources.retrieve_update_for_build(nvr) if updateid is not None: - yield (u'bodhi_update', updateid) + yield ('bodhi_update', updateid) def consume(self, message): """ diff --git a/greenwave/policies.py b/greenwave/policies.py index c67b346..35d8186 100644 --- a/greenwave/policies.py +++ b/greenwave/policies.py @@ -189,7 +189,7 @@ class Rule(yaml.YAMLObject): class RemoteOriginalSpecNvrRule(Rule): - yaml_tag = u'!RemoteOriginalSpecNvrRule' + yaml_tag = '!RemoteOriginalSpecNvrRule' yaml_loader = yaml.SafeLoader def check(self, subject_type, subject_identifier, results, waivers): @@ -231,7 +231,7 @@ class PassingTestCaseRule(Rule): This rule requires either a passing result for the given test case, or a non-passing result with a waiver. """ - yaml_tag = u'!PassingTestCaseRule' + yaml_tag = '!PassingTestCaseRule' yaml_loader = yaml.SafeLoader def check(self, subject_type, subject_identifier, results, waivers): @@ -337,17 +337,17 @@ class PackageSpecificRule(Rule): class FedoraAtomicCi(PackageSpecificRule): - yaml_tag = u'!FedoraAtomicCi' + yaml_tag = '!FedoraAtomicCi' yaml_loader = yaml.SafeLoader class PackageSpecificBuild(PackageSpecificRule): - yaml_tag = u'!PackageSpecificBuild' + yaml_tag = '!PackageSpecificBuild' yaml_loader = yaml.SafeLoader class Policy(yaml.YAMLObject): - yaml_tag = u'!Policy' + yaml_tag = '!Policy' yaml_loader = yaml.SafeLoader def applies_to(self, decision_context, product_version, subject_type): diff --git a/greenwave/resources.py b/greenwave/resources.py index 77c1199..dcdac53 100644 --- a/greenwave/resources.py +++ b/greenwave/resources.py @@ -11,8 +11,8 @@ import json import requests import urllib3.exceptions -import urlparse -import xmlrpclib +from urllib.parse import urlparse, urljoin +import xmlrpc.client from flask import current_app from werkzeug.exceptions import BadGateway @@ -29,7 +29,7 @@ requests_session = requests.Session() @greenwave.utils.retry(wait_on=urllib3.exceptions.NewConnectionError) def retrieve_rev_from_koji(nvr): """ Retrieve cached rev from koji using the nrv """ - proxy = xmlrpclib.ServerProxy(current_app.config['KOJI_BASE_URL']) + proxy = xmlrpc.client.ServerProxy(current_app.config['KOJI_BASE_URL']) build = proxy.getBuild(nvr) if not build: @@ -37,7 +37,7 @@ def retrieve_rev_from_koji(nvr): build, nvr, current_app.config['KOJI_BASE_URL'])) try: - url = urlparse.urlparse(build['extra']['source']['original_url']) + url = urlparse(build['extra']['source']['original_url']) if not url.scheme.startswith('git'): raise BadGateway('Error occurred looking for the "rev" in koji.') return url.fragment @@ -81,8 +81,8 @@ def retrieve_builds_in_update(update_id): 'Assuming no builds in update', update_id) return [] - update_info_url = urlparse.urljoin(current_app.config['BODHI_URL'], - '/updates/{}'.format(update_id)) + update_info_url = urljoin(current_app.config['BODHI_URL'], + '/updates/{}'.format(update_id)) timeout = current_app.config['REQUESTS_TIMEOUT'] verify = current_app.config['REQUESTS_VERIFY'] response = requests_session.get(update_info_url, @@ -97,7 +97,7 @@ def retrieve_update_for_build(nvr): Queries Bodhi to find the update which the given build is in (if any). Returns a Bodhi updateid, or None if the build is not in any update. """ - updates_list_url = urlparse.urljoin(current_app.config['BODHI_URL'], '/updates/') + updates_list_url = urljoin(current_app.config['BODHI_URL'], '/updates/') params = {'builds': nvr} timeout = current_app.config['REQUESTS_TIMEOUT'] verify = current_app.config['REQUESTS_VERIFY'] @@ -140,13 +140,13 @@ def retrieve_results(subject_type, subject_identifier): results = [] if subject_type == 'bodhi_update': results.extend(retrieve_item_results( - {u'type': u'bodhi_update', u'item': subject_identifier})) + {'type': 'bodhi_update', 'item': subject_identifier})) elif subject_type == 'koji_build': - results.extend(retrieve_item_results({u'type': u'koji_build', u'item': subject_identifier})) - results.extend(retrieve_item_results({u'type': u'brew-build', u'item': subject_identifier})) - results.extend(retrieve_item_results({u'original_spec_nvr': subject_identifier})) + results.extend(retrieve_item_results({'type': 'koji_build', 'item': subject_identifier})) + results.extend(retrieve_item_results({'type': 'brew-build', 'item': subject_identifier})) + results.extend(retrieve_item_results({'original_spec_nvr': subject_identifier})) elif subject_type == 'compose': - results.extend(retrieve_item_results({u'productmd.compose.id': subject_identifier})) + results.extend(retrieve_item_results({'productmd.compose.id': subject_identifier})) else: raise RuntimeError('Unhandled subject type %r' % subject_type) return results diff --git a/greenwave/tests/test_resultsdb_consumer.py b/greenwave/tests/test_resultsdb_consumer.py index 968f339..35f0b2a 100644 --- a/greenwave/tests/test_resultsdb_consumer.py +++ b/greenwave/tests/test_resultsdb_consumer.py @@ -10,7 +10,7 @@ def test_announcement_keys_decode_with_list(): cls = greenwave.consumers.resultsdb.ResultsDBHandler app = greenwave.app_factory.create_app() message = {'msg': {'data': { - u'original_spec_nvr'.encode('utf-8'): [u'glibc-1.0-1.fc27'.encode('utf-8')], + 'original_spec_nvr': ['glibc-1.0-1.fc27'], }}} with app.app_context(): @@ -18,14 +18,14 @@ def test_announcement_keys_decode_with_list(): f.return_value = None subjects = list(cls.announcement_subjects(message)) - assert subjects == [(u'koji_build', u'glibc-1.0-1.fc27')] + assert subjects == [('koji_build', 'glibc-1.0-1.fc27')] def test_announcement_subjects_include_bodhi_update(): cls = greenwave.consumers.resultsdb.ResultsDBHandler app = greenwave.app_factory.create_app() message = {'msg': {'data': { - u'original_spec_nvr'.encode('utf-8'): [u'glibc-1.0-2.fc27'.encode('utf-8')], + 'original_spec_nvr': ['glibc-1.0-2.fc27'], }}} with app.app_context(): @@ -47,8 +47,8 @@ def test_announcement_subjects_for_brew_build(): cls = greenwave.consumers.resultsdb.ResultsDBHandler app = greenwave.app_factory.create_app() message = {'msg': {'data': { - u'type'.encode('utf-8'): u'brew-build'.encode('utf-8'), - u'item'.encode('utf-8'): [u'glibc-1.0-3.fc27'.encode('utf-8')], + 'type': 'brew-build', + 'item': ['glibc-1.0-3.fc27'], }}} with app.app_context(): diff --git a/greenwave/utils.py b/greenwave/utils.py index 424d96a..a3635e6 100644 --- a/greenwave/utils.py +++ b/greenwave/utils.py @@ -5,6 +5,7 @@ import glob import logging import os import time +import hashlib import yaml from flask import jsonify, current_app, request @@ -148,3 +149,12 @@ def retry(timeout=None, interval=None, wait_on=Exception): raise # This re-raises the last exception. return inner return wrapper + + +def sha1_mangle_key(key): + """ + Like dogpile.cache.util.sha1_mangle_key, but works correctly on + Python 3 with str keys (which must be encoded to bytes before passing them + to hashlib.sha1()). + """ + return hashlib.sha1(key.encode('utf-8')).hexdigest() diff --git a/setup.py b/setup.py index 12ef1ac..af5a46f 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,6 @@ setup( 'Framework :: Flask', 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', diff --git a/tox.ini b/tox.ini index c61a8f5..7a3169f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = lint,py27,docs +envlist = lint,py36,docs # If the user is missing an interpreter, don't fail skip_missing_interpreters = True From 62f0847bd25ad729f9b9fd0abdadd0b2770b6d2c Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Jun 06 2018 04:02:06 +0000 Subject: [PATCH 3/3] resultsdb consumer: emit messages in consistent order This is mainly for convenience in the tests, so that we don't have to worry about messages being produced in any arbitrary order. But it might be nicer to have messages go onto the bus in a consistent order as well. --- diff --git a/greenwave/consumers/resultsdb.py b/greenwave/consumers/resultsdb.py index 6cd61b1..fb1a3b2 100644 --- a/greenwave/consumers/resultsdb.py +++ b/greenwave/consumers/resultsdb.py @@ -155,8 +155,9 @@ class ResultsDBHandler(fedmsg.consumers.FedmsgConsumer): # For every context X version combination, ask greenwave if this new # result pushes any decisions over a threshold. - for decision_context, product_versions in decision_contexts.items(): - for product_version in product_versions: + for decision_context in sorted(decision_contexts.keys()): + product_versions = decision_contexts[decision_context] + for product_version in sorted(product_versions): greenwave_url = self.fedmsg_config['greenwave_api_url'] + '/decision' data = {