From cbe83d1e855736e474222fb39a3c72d4a529ad1b Mon Sep 17 00:00:00 2001 From: Iryna Shcherbina Date: Dec 06 2017 15:46:04 +0000 Subject: Check the modified dependencies against rawhide repo --- diff --git a/fix_requires.py b/fix_requires.py index 4cabb1f..2155fbe 100644 --- a/fix_requires.py +++ b/fix_requires.py @@ -20,16 +20,19 @@ Notes: TODO: - Check if the package is build for epel/rhel, and then use smth like python%{?fedora:2}-foo in requires if not available. +- Alert packagers if any of the changed requires is not available for +lower versions of Fedora (check for 26 and 27) """ import logging -import re import pathlib import tempfile import click from fixrequires import LocalRepo, PagureFork +from fixrequires.dnf_info import DNFInfo +from fixrequires.modifier import RequiresModifier from qa import test_in_mock, test_in_koji @@ -40,20 +43,6 @@ def get_logger(name=__name__, level=logging.DEBUG): return logger -RHEL_MARKERS = ('{?rhel}', '{?el6}', '{?epel7}') -REQUIRES_PATTERN = '#?\s*(Build)?Requires:\s+(.*)' - -# pattern: (what to replace, how to replace, what to prepend) -REPLACE_PATTERNS = { - # Special cases are sometimes special enough. - '^PyYAML$': ('^PyYAML$', 'python2-pyyaml', ''), - # Common cases. - '^python-\w*': ('^python-', 'python2-', ''), - '[^/]*-python-\w*': ('-python-', '-python2-', ''), - '[^/]*-python$': ('-python$', '', 'python2-'), - '^python$': ('^python$', 'python2', ''), -} - PAGURE_INSTANCE = 'https://src.fedoraproject.org' FINALIZING_DOC = 'https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3' @@ -77,43 +66,7 @@ PR_DESCRIPTION = ( ) -def fix_requires_line(requires): - requires = re.split('([\s\,])', requires) - modified_requires = [] - - for require in requires: - for pattern, (replace_what, replace_with, prepend) in REPLACE_PATTERNS.items(): - if re.match(pattern, require): - require = re.sub( - replace_what, replace_with, require) - require = prepend + require.lower() - modified_requires.append(require) - - return ''.join(modified_requires) - - -def fix_spec(spec): - modified_spec = [] - - for index, line in enumerate(spec.split('\n')): - match = re.match(REQUIRES_PATTERN, line) - if match: - for requires in match.groups(): - if not requires or requires == 'Build': - continue - - modified_requires = fix_requires_line(requires) - - if requires != modified_requires: - line = line.replace(requires, modified_requires) - - modified_spec.append(line) - - modified_spec = '\n'.join(modified_spec) - return modified_spec - - -def fix_specfile(specfile): +def fix_specfile(specfile, dnf_info): """Fix deps in a spec file. Return: True if fixed, False if no changes were made @@ -121,7 +74,7 @@ def fix_specfile(specfile): with open(specfile, 'rt') as f: spec = f.read() - new_spec = fix_spec(spec) + new_spec = RequiresModifier(dnf_info).run(spec) with open(specfile, 'wt') as out: out.write(new_spec) @@ -161,9 +114,15 @@ def fix_specfile(specfile): '--fas-password', help="FAS password (needed for PR)", prompt=True, hide_input=True, confirmation_prompt=False) +@click.option( + '-v', '--verbose', + help="Verbose", is_flag=True) def main(packages, dirname, no_mock_build, no_koji_build, pagure, - pagure_token, pagure_user, fas_user, fas_password): - logger = get_logger() + pagure_token, pagure_user, fas_user, fas_password, verbose): + if verbose: + logger = get_logger(level=logging.DEBUG) + else: + logger = get_logger(level=logging.ERROR) with open(packages, 'rt') as f: require_misnamed = f.read().splitlines() @@ -173,6 +132,8 @@ def main(packages, dirname, no_mock_build, no_koji_build, pagure, if not dirname: dirname = tempfile.mkdtemp() + dnf_info = DNFInfo(logger) + non_fedora_packages = [] fixed_packages = [] problem_packages = [] @@ -202,7 +163,7 @@ def main(packages, dirname, no_mock_build, no_koji_build, pagure, local_repo.clone() local_repo.create_branch(GIT_BRANCH) - spec_fixed = fix_specfile(local_repo.specfile) + spec_fixed = fix_specfile(local_repo.specfile, dnf_info) if not spec_fixed: raise Exception( 'No changes made to the spec file. ' diff --git a/fixrequires/dnf_info.py b/fixrequires/dnf_info.py new file mode 100644 index 0000000..e4cf3d1 --- /dev/null +++ b/fixrequires/dnf_info.py @@ -0,0 +1,49 @@ +""" +Python DNF API integration to verify +that the modified dependencies are there. +""" + +import dnf + + +PYTHON2_SEED = [ + 'python-devel', 'python2-devel', 'python', 'python2', 'python-libs', + 'python(abi) = 2.7', '/usr/bin/python', '/usr/bin/python2', + '/usr/bin/python2.7', 'libpython2.7.so.1.0', 'libpython2.7.so.1.0()(64bit)', + 'pygtk2', 'pygobject2', 'pycairo'] + +METALINK = 'https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch' + + +class DNFInfo(object): + + def __init__(self, logger): + self.logegr = logger + self.rawhide_query = self._get_query('rawhide') + self.all_py2_packages = self._get_all_py2_packages() + + def _get_query(self, releasever): + base = dnf.Base() + base.conf.substitutions['releasever'] = releasever + repo = dnf.repo.Repo(releasever, parent_conf=base.conf) + repo.metalink = dnf.conf.parser.substitute(METALINK, base.conf.substitutions) + base.repos.add(repo) + repo.skip_if_unavailable = False + repo.enable() + repo.load() + base.fill_sack(load_system_repo=False, load_available_repos=True) + return base.sack.query() + + def _get_all_py2_packages(self): + self.logger.debug('Preparation: getting Python 2 packages') + return [pkg.name for pkg in + self.rawhide_query.filter(requires=PYTHON2_SEED).run()] + + def requires_py2(self, require_name): + pkg = self.rawhide_query.filter(provides=require_name) + if pkg.count(): + pkg = pkg.run()[0] + return pkg.name in self.all_py2_packages + + def is_in_rawhide(self, require_name): + return self.rawhide_query.filter(provides=require_name).count() diff --git a/fixrequires/modifier.py b/fixrequires/modifier.py new file mode 100644 index 0000000..fa0fa0f --- /dev/null +++ b/fixrequires/modifier.py @@ -0,0 +1,78 @@ +""" +""" +import re + +RHEL_MARKERS = ('{?rhel}', '{?el6}', '{?epel7}') +REQUIRES_PATTERN = '#?\s*(Build)?Requires:\s+(.*)' + +# pattern: (what to replace, how to replace, what to prepend) +REPLACE_PATTERNS = { + # Special cases are sometimes special enough. + '^PyYAML$': ('^PyYAML$', 'python2-pyyaml', ''), + # Common cases. + '^python-\w*': ('^python-', 'python2-', ''), + '[^/]*-python-\w*': ('-python-', '-python2-', ''), + '[^/]*-python$': ('-python$', '', 'python2-'), + '^python$': ('^python$', 'python2', ''), +} + + +class ModifierException(Exception): + """Base Exception for RequiresModifier.""" + + +class RequiresModifier(object): + + def __init__(self, dnf_info): + self.dnf_info = dnf_info + + def run(self, spec): + return self._fix_spec(spec) + + def _fix_spec(self, spec): + modified_spec = [] + + for index, line in enumerate(spec.split('\n')): + match = re.match(REQUIRES_PATTERN, line) + if match: + for requires in match.groups(): + if not requires or requires == 'Build': + continue + + modified_requires = self._fix_requires_line(requires) + + if requires != modified_requires: + line = line.replace(requires, modified_requires) + + modified_spec.append(line) + + modified_spec = '\n'.join(modified_spec) + return modified_spec + + def _fix_requires_line(self, requires): + requires = re.split('([\s\,])', requires) + modified_requires = [] + + for require in requires: + for pattern, (replace_what, replace_with, prepend) in REPLACE_PATTERNS.items(): + if re.match(pattern, require): + + # If the package is named python-smth but does not depend on + # python2, then we do not care about it (e.g. python-rpm-macros) + if not self.dnf_info.requires_py2(require): + continue + + new_require = re.sub( + replace_what, replace_with, require) + new_require = prepend + new_require.lower() + + if self.dnf_info.is_in_rawhide(new_require): + require = new_require + else: + raise ModifierException( + f'{require} should be changed to {new_require}, ' + 'but seems like it was converted wrong!') + + modified_requires.append(require) + + return ''.join(modified_requires) diff --git a/fixrequires/pagure_fork.py b/fixrequires/pagure_fork.py index b4ed8c3..ef82791 100644 --- a/fixrequires/pagure_fork.py +++ b/fixrequires/pagure_fork.py @@ -34,14 +34,14 @@ class PagureFork(object): self.fork_api = f"{self.pagure.instance}/api/0/fork" def do_fork(self): - self.logger .debug(f"Creating fork of {self.package_name} for user {self.pagure_user}") + self.logger.debug(f"Creating fork of {self.package_name} for user {self.pagure_user}") try: payload = {'wait': True, 'namespace': 'rpms', 'repo': self.package_name} response = self.pagure._call_api(self.fork_api, method='POST', data=payload) - self.logger .debug(f"Fork created: {response}") + self.logger.debug(f"Fork created: {response}") except APIError as err: if 'already exists' in str(err): - self.logger .info( + self.logger.info( f'User {self.pagure_user} already has a fork of {self.package_name}') else: raise err diff --git a/qa/__init__.py b/qa/__init__.py index 0e61fc7..cd51feb 100644 --- a/qa/__init__.py +++ b/qa/__init__.py @@ -18,4 +18,4 @@ from .build_koji import test_in_koji # build_in_mock(package_dirname) # koji_scratch_build = build_in_koji(package_dirname) -# return koji_scratch_build \ No newline at end of file +# return koji_scratch_build