From 5b7101e8bd0291df14503800969e5f3613fced05 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 17 2017 14:36:17 +0000 Subject: [PATCH 1/12] Split pypi requirements and refine versions Signed-off-by: Chenxiong Qi --- diff --git a/requirements/pypi.txt b/requirements/pypi.txt index fe6a687..5a5c224 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -1,8 +1,8 @@ # List of Python packages that can be installed from PyPI. cccolutils >= 1.4 -GitPython >= 0.2.0 -pycurl >= 7.43 +GitPython >= 0.3.2 +pycurl >= 7.19 requests rpm-py-installer six >= 1.9.0 @@ -10,17 +10,8 @@ six >= 1.9.0 # openidc-client # used for MBS OIDC authentication # requests-kerberos # used for MBS Kerberos authentication - # Only required for <= Python 2.6 -argparse == 1.4.0 - -# For running tests -coverage == 4.4.1 -flake8 >= 2.5.5 -mock >= 2.0.0 -nose >= 1.3.7 -git+https://pagure.io/rpmfluff.git@0.5.1#egg=rpmfluff -openidc-client # used in MBS tests +argparse == 1.4.0 ; python_version < '2.7' # Several package that are not available in PyPI are also listed here. # If rpkg runs from a Python virtualenv, you may need --site-packages to create @@ -30,4 +21,3 @@ openidc-client # used in MBS tests # system. # # koji -# mock diff --git a/requirements/test-pypi.txt b/requirements/test-pypi.txt new file mode 100644 index 0000000..4747c91 --- /dev/null +++ b/requirements/test-pypi.txt @@ -0,0 +1,11 @@ +-r pypi.txt + +coverage +flake8 == 3.2.0 +mock == 1.0.1 +nose == 1.3.7 + +# used in MBS tests +openidc-client + +git+https://pagure.io/rpmfluff.git@0.5.1#egg=rpmfluff From 59ba6a7f8fc45fc9764aab358ff655985e5b45be Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 17 2017 14:36:17 +0000 Subject: [PATCH 2/12] Set install and tests requires in setup.py Signed-off-by: Chenxiong Qi --- diff --git a/setup.py b/setup.py index ddbb6fd..1038501 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,38 @@ #!/usr/bin/python +import os + from setuptools import setup, find_packages +def read_requirements(requirements_file): + specifiers = [] + dep_links = [] + + def read_lines(): + with open(requirements_file, 'r') as f: + for line in f: + line = line.strip() + if line == '' or line.startswith('#') or line.startswith('-r'): + continue + yield line + + for line in read_lines(): + if line.startswith('git+'): + dep_links.append(line) + else: + specifiers.append(line) + + return specifiers, dep_links + +setup_py_path = os.path.dirname(os.path.realpath(__file__)) +pypi_txt = os.path.join(setup_py_path, 'requirements', 'pypi.txt') +test_pypi_txt = os.path.join(setup_py_path, 'requirements', 'test-pypi.txt') + +install_requires, dep_links = read_requirements(pypi_txt) +tests_require, test_dep_links = read_requirements(test_pypi_txt) +dep_links += test_dep_links + setup( name="rpkg", version="1.51", @@ -13,8 +43,9 @@ setup( license="GPLv2+", url="https://pagure.io/rpkg", packages=find_packages(), - install_requires=['six', 'pycurl', 'cccolutils'], # + koji, but it's not in PyPI - tests_require=['nose', 'mock', 'GitPython'], + install_requires=install_requires, + tests_require=tests_require, + dependency_links=dep_links, test_suite='nose.collector', classifiers=( 'Development Status :: 5 - Production/Stable', From 6fc7100f5bc27ac9ebece5358b12e87919f5f9e8 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 19 2017 05:05:31 +0000 Subject: [PATCH 3/12] Install Koji shared library via setuptools Since version 1.15.0, Koji supports setuptools. This patch make it possible to install koji into rpkg Python env in order to build a development environment as well as provisioning an environment to run tests. Signed-off-by: Chenxiong Qi --- diff --git a/requirements/pypi.txt b/requirements/pypi.txt index 5a5c224..1a7c3aa 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -13,11 +13,15 @@ six >= 1.9.0 # Only required for <= Python 2.6 argparse == 1.4.0 ; python_version < '2.7' +# setuptools support is supported since version 1.15. +# Requires koji>=1.15.0 +# Install koji from source code and tag koji-1.15.0 without waiting for Koji +# developers to publish it to PyPI. +git+https://pagure.io/koji.git@koji-1.15.0#egg=koji + # Several package that are not available in PyPI are also listed here. # If rpkg runs from a Python virtualenv, you may need --site-packages to create # the environment so that following Python modules can be imported. # # Please see also fedora-py2.txt or fedora-py3.txt to install them in your # system. -# -# koji From b671d18b8f282399fe5d276a6a401575ac5f706d Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 19 2017 05:11:40 +0000 Subject: [PATCH 4/12] Add files under requirements/ to sdist package Signed-off-by: Chenxiong Qi --- diff --git a/MANIFEST.in b/MANIFEST.in index 3f8b370..37cd5e2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,6 @@ include doc/rpkg_man_page.py include bin/rpkg recursive-include tests * recursive-include etc * + +include requirements/README.rst +include requirements/*.txt From 5af6da19a64d56b550e6388befb8896b59a78716 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 19 2017 05:16:25 +0000 Subject: [PATCH 5/12] Use flake8 3.5.0 Signed-off-by: Chenxiong Qi --- diff --git a/requirements/test-pypi.txt b/requirements/test-pypi.txt index 4747c91..99fd93f 100644 --- a/requirements/test-pypi.txt +++ b/requirements/test-pypi.txt @@ -1,7 +1,7 @@ -r pypi.txt coverage -flake8 == 3.2.0 +flake8 == 3.5.0 mock == 1.0.1 nose == 1.3.7 diff --git a/tox.ini b/tox.ini index 9729792..138151d 100644 --- a/tox.ini +++ b/tox.ini @@ -11,5 +11,5 @@ commands = nosetests {posargs} [testenv:flake8] basepython = python3 -deps = flake8 >= 2.5.5 +deps = flake8 == 3.5.0 commands = flake8 pyrpkg/ tests/ From 834cb71f3f4a0099b4fb776c0cebbd02539b4be8 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 27 2017 12:04:51 +0000 Subject: [PATCH 6/12] Run tox to run tests and check code styles Fix #276 Signed-off-by: Chenxiong Qi --- diff --git a/pip-pycurl b/pip-pycurl new file mode 100755 index 0000000..1614619 --- /dev/null +++ b/pip-pycurl @@ -0,0 +1,16 @@ +#!/bin/bash + +if python -c "import pycurl" &>/dev/null; then + exit 0 +fi + +# We need to build pycurl with openssl in Fedora 27, otherwise nss should be +# used. +# See also: https://fedoraproject.org/wiki/Changes/libcurlBackToOpenSSL + +if [ "$(rpm --eval '%{dist}')" == ".fc27" ]; then + install_option="--with-openssl" +else + install_option="--with-nss" +fi +pip install -v -I --install-option="${install_option}" "pycurl>=7.19" diff --git a/tox.ini b/tox.ini index 138151d..33649e8 100644 --- a/tox.ini +++ b/tox.ini @@ -2,12 +2,12 @@ envlist = py27,py35,flake8 [testenv] -sitepackages = True -deps = -r{toxinidir}/requirements/pypi.txt -whitelist_externals = - flake8 - nosetests -commands = nosetests {posargs} +skip_install = True +deps = + -r{toxinidir}/requirements/test-pypi.txt +commands = + {toxinidir}/pip-pycurl + nosetests {posargs} [testenv:flake8] basepython = python3 From c20be5152d1742962a2c8b673791232f0b13f578 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 27 2017 12:04:51 +0000 Subject: [PATCH 7/12] Add py36 to testenv Fix #274 Signed-off-by: Chenxiong Qi --- diff --git a/tox.ini b/tox.ini index 33649e8..ba22712 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py35,flake8 +envlist = py27,py35,py36,flake8 [testenv] skip_install = True From 2c4acf42013af976f2bd0bbeccf246eadd111568 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 27 2017 12:04:51 +0000 Subject: [PATCH 8/12] Fix tests: not impact by dict.items call Signed-off-by: Chenxiong Qi --- diff --git a/tests/test_cli.py b/tests/test_cli.py index b6d1bf2..01e24a0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1944,7 +1944,7 @@ class TestModulesCli(CliTestCase): 'module-builds/2150?verbose=true') mock_get.assert_called_once_with(exp_url, timeout=60) output = sys.stdout.getvalue().strip() - expected_output = """ + expected_output = """\ Name: python3-ecosystem Stream: master Version: 20171010145511 @@ -1969,8 +1969,10 @@ Components: NVR: None State: FAILED Koji Task: -""".strip() # noqa: W291 - self.assertEqual(expected_output, output) +""" + self.maxDiff = None + self.assertEqual(self.sort_lines(expected_output), + self.sort_lines(output)) @patch('sys.stdout', new=StringIO()) @patch.object(Commands, 'kojiweburl', @@ -2003,7 +2005,7 @@ Components: mock_get.assert_called_once_with(exp_url, timeout=60) mock_system.assert_called_once_with('clear') output = sys.stdout.getvalue().strip() - expected_output = """ + expected_output = """\ Failed: module-build-macros https://koji.fedoraproject.org/koji/taskinfo?taskID=22370514 python-dns @@ -2012,8 +2014,10 @@ Failed: Summary: 3 components in the "failed" state torsava's build #2150 of python3-ecosystem-master is in the "failed" state (reason: Some error) (koji tag: "module-14050f52e62d955b") -""".strip() # noqa: E501 - self.assertEqual(output, expected_output) +""" + self.maxDiff = None + self.assertEqual(self.sort_lines(expected_output), + self.sort_lines(output)) @patch('sys.stdout', new=StringIO()) @patch('requests.get') diff --git a/tests/utils.py b/tests/utils.py index 7e77912..0993ee2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,6 +4,7 @@ import os import subprocess import tempfile import shutil +import six import sys from pyrpkg import Commands @@ -211,3 +212,11 @@ class CommandTestCase(Assertions, Utils, unittest.TestCase): self.write_file(_filename, file_content or 'Hello rpkg') repo.index.add([_filename]) repo.index.commit(commit_message or 'update document') + + @staticmethod + def sort_lines(s): + buf = six.moves.StringIO(s) + try: + return sorted((line.strip() for line in buf)) + finally: + buf.close() \ No newline at end of file From 238ec3207436ced7a23b2fb637820123c7281c56 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 27 2017 12:04:51 +0000 Subject: [PATCH 9/12] Fix tests that do not work with Python 3 Signed-off-by: Chenxiong Qi --- diff --git a/tests/test_cli.py b/tests/test_cli.py index 01e24a0..6e7cb04 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -947,14 +947,14 @@ class TestNew(CliTestCase): # diff is return from Commands.new as bytestring when using # GitPython<1.0. So, mock new method directly to test diff in # bytestring can be printed correctly. - new.return_value = b'New content' + new.return_value = 'New content' cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'new'] with patch('sys.argv', new=cli_cmd): cli = self.new_cli() cli.new() output = sys.stdout.getvalue() - self.assertTrue(b'New content' in output) + self.assertTrue('New content' in output) class TestNewPrintUnicode(CliTestCase): @@ -1625,7 +1625,7 @@ class TestPatch(CliTestCase): cli_cmd = ['rpkg', '--path', self.cloned_repo_path, 'patch', 'fix'] with patch('sys.argv', new=cli_cmd): cli = self.new_cli() - with patch('__builtin__.open', mock_open()) as m: + with patch.object(six.moves.builtins, 'open', mock_open()) as m: cli.patch() m.return_value.write.assert_called_once_with('+ diff') @@ -1669,8 +1669,8 @@ class TestPatch(CliTestCase): cli.cmd.ver) copied_patch_file = '{0}~'.format(patch_file) - with patch('__builtin__.open', - mock_open(read_data=origin_diff)) as m: + with patch.object(six.moves.builtins, 'open', + mock_open(read_data=origin_diff)) as m: with patch('os.path.exists', return_value=True) as exists: cli.patch() diff --git a/tests/test_commands.py b/tests/test_commands.py index 86791f5..cb6c757 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -776,7 +776,7 @@ class TestConfigMockConfigDir(CommandTestCase): @contextmanager def assert_file_op(self, filename, mode, write_data=None): """Assert file object operation""" - with patch('__builtin__.open', mock_open()) as mock: + with patch.object(six.moves.builtins, 'open', mock_open()) as mock: yield mock.assert_called_once_with(filename, mode) if write_data is not None: @@ -834,7 +834,7 @@ class TestConfigMockConfigDir(CommandTestCase): mock.assert_called_once_with(None) def test_fail_if_error_occurs_while_writing_cfg_file(self): - with patch('__builtin__.open', mock_open()) as m: + with patch.object(six.moves.builtins, 'open', mock_open()) as m: m.return_value.write.side_effect = IOError with patch('pyrpkg.Commands._cleanup_tmp_dir') as mock: @@ -875,7 +875,7 @@ class TestConfigMockConfigDirWithNecessaryFiles(CommandTestCase): def test_create_empty_cfg_files_if_not_exist_in_system_mock(self, exists): cmd = self.make_commands() - with patch('__builtin__.open', mock_open()) as m: + with patch.object(six.moves.builtins, 'open', mock_open()) as m: cmd._config_dir_other('/path/to/config-dir') m.assert_has_calls([ @@ -903,7 +903,7 @@ class TestConfigMockConfigDirWithNecessaryFiles(CommandTestCase): def test_fail_if_error_when_write_empty_cfg_files(self, exists): cmd = self.make_commands() - with patch('__builtin__.open', mock_open()) as m: + with patch.object(six.moves.builtins, 'open', mock_open()) as m: m.side_effect = IOError self.assertRaises(rpkgError, cmd._config_dir_other, '/path/to/config-dir') From 0f235a691e42f9c666ce6e42eec2f3252132445f Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 27 2017 12:04:51 +0000 Subject: [PATCH 10/12] Fix flake8 errors Signed-off-by: Chenxiong Qi --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 3757fcf..6557b90 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -398,7 +398,7 @@ class Commands(object): try: session = koji.ClientSession(koji_config['server'], session_opts) - except: + except Exception: raise rpkgError('Could not initiate %s session' % os.path.basename(self.build_client)) else: if anon: @@ -1070,7 +1070,7 @@ class Commands(object): # anyway. if int(re.search(r'\d+', self.distval).group()) < 6: return('md5') - except: + except Exception: # An error here is OK, don't bother the user. pass @@ -1505,7 +1505,7 @@ class Commands(object): try: output = subprocess.check_output(cmd) hash = output.split()[0] - except: + except Exception: # don't do anything here, we'll handle not having hash # later pass @@ -1738,7 +1738,7 @@ class Commands(object): # see if our branch is tracking anything try: self.load_branch_merge() - except: + except Exception: self.log.warning('Current branch cannot be pushed anywhere!') untracked_patches = self.find_untracked_patches() diff --git a/tests/test_cli.py b/tests/test_cli.py index 6e7cb04..4edc1cf 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1969,7 +1969,7 @@ Components: NVR: None State: FAILED Koji Task: -""" +""" # noqa self.maxDiff = None self.assertEqual(self.sort_lines(expected_output), self.sort_lines(output)) @@ -2014,7 +2014,7 @@ Failed: Summary: 3 components in the "failed" state torsava's build #2150 of python3-ecosystem-master is in the "failed" state (reason: Some error) (koji tag: "module-14050f52e62d955b") -""" +""" # noqa self.maxDiff = None self.assertEqual(self.sort_lines(expected_output), self.sort_lines(output)) diff --git a/tests/utils.py b/tests/utils.py index 0993ee2..46c2b79 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -219,4 +219,4 @@ class CommandTestCase(Assertions, Utils, unittest.TestCase): try: return sorted((line.strip() for line in buf)) finally: - buf.close() \ No newline at end of file + buf.close() From e2fea4a57f74acdb2a26233cec088c2e58ecca57 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Dec 28 2017 05:21:58 +0000 Subject: [PATCH 11/12] Declare Python versions rpkg can work with Fix #278 Signed-off-by: Chenxiong Qi --- diff --git a/README.rst b/README.rst index adf8cbd..bed7669 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,8 @@ This is the rpkg project, which mostly is a python library for dealing with rpm packaging in a git source control. pyrpkg is the base library that sites can subclass to create useful tools. +rpkg now can work with Python 2.6, 2.7, 3.5 and 3.6. + License ======= diff --git a/setup.py b/setup.py index 1038501..ad9d764 100755 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ setup( 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Build Tools', 'Topic :: Software Development :: Libraries :: Python Modules', ), From 0218bb284810af1618051cb656df8ca6a421d836 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Jan 09 2018 03:44:00 +0000 Subject: [PATCH 12/12] Add dependent packages for Python 2.6 in setup.py Signed-off-by: Chenxiong Qi --- diff --git a/requirements/pypi.txt b/requirements/pypi.txt index 1a7c3aa..8a4d04e 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -10,8 +10,8 @@ six >= 1.9.0 # openidc-client # used for MBS OIDC authentication # requests-kerberos # used for MBS Kerberos authentication -# Only required for <= Python 2.6 -argparse == 1.4.0 ; python_version < '2.7' +# There are dependent packages for running with Python 2.6, and those will be +# handled in setup.py. # setuptools support is supported since version 1.15. # Requires koji>=1.15.0 diff --git a/setup.py b/setup.py index ad9d764..a9d01fa 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/python import os +import sys from setuptools import setup, find_packages @@ -25,6 +26,7 @@ def read_requirements(requirements_file): return specifiers, dep_links + setup_py_path = os.path.dirname(os.path.realpath(__file__)) pypi_txt = os.path.join(setup_py_path, 'requirements', 'pypi.txt') test_pypi_txt = os.path.join(setup_py_path, 'requirements', 'test-pypi.txt') @@ -33,6 +35,12 @@ install_requires, dep_links = read_requirements(pypi_txt) tests_require, test_dep_links = read_requirements(test_pypi_txt) dep_links += test_dep_links +ver = sys.version_info +if ver.major <= 2 and ver.minor < 7: + install_requires.append('argparse==1.4.0') + tests_require.append('unittest2') + + setup( name="rpkg", version="1.51",