From a0e737d109582b286a29b46ce19057a0f480967e Mon Sep 17 00:00:00 2001 From: Iryna Shcherbina Date: Nov 29 2017 20:18:17 +0000 Subject: Initial commit --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/fix_requires.py b/fix_requires.py new file mode 100644 index 0000000..bcfc129 --- /dev/null +++ b/fix_requires.py @@ -0,0 +1,264 @@ +#!/usr/bin/python3 +""" +This script will take packages from a file supplied and: + +- clone the spec file to the temp directory +- fix the misnamed requirements +- build an srpm +- run a mock build +- run a Koji scratch build +- create a fork of the repo in Pagure (for a specified user) +- push changes to a fork +- create a Pagure Pull Request from fork to upstream + +Notes: +- For Koji scratch build and to work and to create a PR + you'll have to be "kinited" with your FAS +- To create a fork, you will need to provide your Pagure + username and api key (can be created on Pagure UI on user settings) + +TODO: +- Check if the package is build for epel/rhel, and then use smth like +python%{?fedora:2}-foo in requires if not available. +""" + +import logging +import re +import pathlib +import tempfile +import shutil + +import click + +from fixrequires import LocalRepo, PagureFork +from qa import test_in_mock, test_in_koji + + +def get_logger(name=__name__, level=logging.DEBUG): + logging.basicConfig(format='%(levelname)s: %(message)s') + logger = logging.getLogger(name) + logger.setLevel(level) + return logger + + +RHEL_MARKERS = ('{?rhel}', '{?el6}', '{?epel7}') +REQUIRES_PATTERN = '#?\s*(Build)?Requires:\s+(.*)' +REPLACE_PATTERNS = { + # pattern: (what to replace, how to replace) + '^python-\w*': ('python-', 'python2-'), + '[!/]*-python-\w*': ('-python-', 'python2-'), + '[!/]*-python,?$': ('-python', 'python2-'), + '^python,?$': ('python', 'python2'), + # Special cases are sometimes special enough. + '^PyYAML,?$': ('PyYAML', 'python2-pyyaml'), +} + +PAGURE_INSTANCE = 'https://src.fedoraproject.org' +FINALIZING_DOC = 'https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3' + +GIT_BRANCH = 'pymisnamed' +COMMIT = 'Fix misnamed Python 2 dependencies declarations' +COMMENT = f'{COMMIT}\n (See {FINALIZING_DOC})' + +PR_DESCRIPTION = ( + 'This package uses names with ambiguous `python-` prefix in requirements.\n\n' + 'According to Fedora Packaging guidelines for Python [0], ' + 'packages must use names with either `python2-` or `python3-` ' + 'prefix in requirements where available.\n\n' + 'This PR is part of Fedora\'s Switch to Python 3 effort [1] ' + 'aiming to fix misnamed dependencies declarations across Python packages.\n\n' + 'Note that, although this PR was created automatically, any comments or issues which ' + 'you might find with it during the review will be fixed.' + 'The PR will remain open for review for a week, and ' + 'if no feedback received will be merged.\n\n' + '[0] https://fedoraproject.org/wiki/Packaging:Python#Dependencies\n' + f'[1] {FINALIZING_DOC}' +) + + +def fix_requires_line(requires): + requires = re.split('(\s)', requires) + modified_requires = [] + + for require in requires: + for pattern, (replace_what, replace_with) in REPLACE_PATTERNS.items(): + if re.match(pattern, require): + require = require.replace( + replace_what, + replace_with) + modified_requires.append(require) + + return ''.join(modified_requires) + + +def fix_spec(spec): + modified_spec = [] + + for index, line in enumerate(spec.split('\n')): + match = re.match(REQUIRES_PATTERN, line) + if match: + for requires in match.groups(): + if not requires or requires == 'Build': + continue + + modified_requires = fix_requires_line(requires) + + if requires != modified_requires: + line = line.replace(requires, modified_requires) + + modified_spec.append(line) + + modified_spec = '\n'.join(modified_spec) + return modified_spec + + +def fix_specfile(specfile): + with open(specfile, 'rt') as f: + spec = f.read() + + new_spec = fix_spec(spec) + + with open(specfile, 'wt') as out: + out.write(new_spec) + + +@click.command(help=__doc__) +@click.option( + '--packages', + help="A file with package names to process (separated by new line)", + default=None, type=click.Path(exists=True)) +@click.option( + '--dirname', + help="Directory path to clone packages and do the tests", + default=None, type=click.Path(exists=True)) +@click.option( + '--cleandir', + help="Clean the directory if not empty", is_flag=True) +@click.option( + '--no-mock-build', + help="Do not do any builds/tests in mock", is_flag=True) +@click.option( + '--no-koji-build', + help="Do not do any builds/tests in Koji", is_flag=True) +@click.option( + '--pagure', + help="Create pagure fork for project, push to fork and create PR", + is_flag=True) +@click.option( + '--pagure-token', + help="Pagure token to create a fork", default=None) +@click.option( + '--pagure-user', + help="Pagure user to create fork for", default=None) +@click.option( + '--fas-user', + help="FAS username (needed for PR)", default=None) +@click.option( + '--fas-password', + help="FAS password (needed for PR)", + prompt=True, hide_input=True, confirmation_prompt=False) +def main(packages, dirname, cleandir, no_mock_build, no_koji_build, pagure, + pagure_token, pagure_user, fas_user, fas_password): + logger = get_logger() + with open(packages, 'rt') as f: + require_misnamed = f.read().splitlines() + + if not require_misnamed: + logger.info('No packages with misnamed requires found') + + if not dirname: + dirname = tempfile.mkdtemp() + if cleandir: + shutil.rmtree(f'{dirname}') + + non_fedora_packages = [] + fixed_packages = [] + problem_packages = [] + + logger.debug(f'Cloning packages into {dirname}') + for package_name in require_misnamed: + + try: + local_repo = LocalRepo( + package_name, + pathlib.Path(f'{dirname}/{package_name}'), + logger) + pagure_fork = PagureFork( + package_name, pagure_token, pagure_user, logger) + + # Check if the package has EPEL branches. And skip those in the + # first batches, then create some clever scheme to handle them. + # TODO: fix this not to send the request to pagure, but check branches + # in local repo. + if pagure_fork.has_epel_branch(): + logger.info( + f'The package {package_name} seems to be built for Fedora ' + 'and EPEL. Skipping for now as the spec file may be shared.') + non_fedora_packages.append(package_name) + continue + + local_repo.clone() + local_repo.create_branch(GIT_BRANCH) + + fix_specfile(local_repo.specfile) + + local_repo.bump_spec(COMMENT) + local_repo.show_diff(local_repo.specfile) + + if pagure: + # Create a fork before testing to avoid + # waiting the fork to be ready to be pushed to. + if not pagure_token or not pagure_user: + raise Exception("Please provide both pagure user and token") + break + + pagure_fork.do_fork() + fork_url = pagure_fork.get_ssh_git_url() + + # QA. + local_repo.create_srpm() + if not no_mock_build: + test_in_mock(local_repo, logger) + + if not no_koji_build: + koji_scratch_build = test_in_koji(local_repo, logger) + else: + koji_scratch_build = '' + + # All good at this point. Go ahead and push. + if pagure: + # Use username as a name for fork remote if not provided. + remote_name = pagure_user + local_repo.add_to_git_remotes(remote_name, fork_url) + local_repo.git_add(local_repo.specfile) + local_repo.git_commit(COMMIT) + local_repo.git_push(remote_name, GIT_BRANCH) + + pagure_fork.create_pull_request( + from_branch=GIT_BRANCH, + to_branch='master', + title=COMMIT, + description=PR_DESCRIPTION + f'\n\nKoji scratch build: {koji_scratch_build}', + fas_user=fas_user, fas_password=fas_password) + + except Exception as err: + logger.error(f"Failed to fix a package {package_name}. Error: {err}") + problem_packages.append(package_name) + continue + + fixed_packages.append(package_name) + + result = ( + '\n\nRESULTS:\n' + f'The following {len(non_fedora_packages)} were skipped,' + f' because they are also built for EPEL:\n{non_fedora_packages}\n' + f'The following {len(problem_packages)} packages had problem' + f' while testing and were not pushed:\n{problem_packages}\n.' + f'The following {len(fixed_packages)} packages were successfully fixed:\n' + f'{fixed_packages}\n.' + ) + logger.info(result) + + +if __name__ == '__main__': + main() diff --git a/fixrequires/__init__.py b/fixrequires/__init__.py new file mode 100644 index 0000000..ccf6c13 --- /dev/null +++ b/fixrequires/__init__.py @@ -0,0 +1,5 @@ +""" +""" + +from .local_repo import LocalRepo +from .pagure_fork import PagureFork diff --git a/fixrequires/local_repo.py b/fixrequires/local_repo.py new file mode 100644 index 0000000..938ca5f --- /dev/null +++ b/fixrequires/local_repo.py @@ -0,0 +1,137 @@ +""" +""" + +import os +import subprocess +import time + + +class LocalRepoException(Exception): + + """Base exception class for LocalRepo. + """ + + +class LocalRepo(object): + + """ + """ + + def __init__(self, package_name, dirname, logger): + self.package_name = package_name + self.dirname = dirname + self.logger = logger + + self._specfile = None + + @property + def specfile(self): + if not self._specfile: + self._specfile, = self.dirname.glob('*.spec') + return self._specfile + + # Fedpkg commands. + def clone(self): + self.logger.debug(f'Cloning {self.package_name} into {self.dirname}') + subprocess.check_output( + ['fedpkg', 'clone', self.package_name, f'{self.dirname}'], # TODO. + stderr=open(os.devnull, 'w')) + + def create_srpm(self): + subprocess.call( + ['fedpkg', '--release', 'master', 'srpm'], + cwd=self.dirname, stdout=subprocess.PIPE) + srpm_name = self.dirname.glob('*.src.rpm') + return srpm_name + + def run_mockbuild(self): + try: + output = subprocess.check_output( + ['fedpkg', '--release', 'master', 'mockbuild'], + cwd=self.dirname, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + self.logger.error( + f'Mock build did not pass for {self.package_name}. ' + f'Error: {err.output}') + raise err + return output + + def run_koji_scratch_build(self): + try: + output = subprocess.check_output( + ['fedpkg', '--release', 'master', 'build', '--scratch', '--srpm'], cwd=self.dirname) + except subprocess.CalledProcessError as err: + self.logger.error( + f'Failed to run Koji scratch build for {self.dirname}. ' + f'Error: {err.output}') + raise err + return output + + def bump_spec(self, comment): + subprocess.check_call( + ['rpmdev-bumpspec', '-c', comment, self.specfile]) + + # Git commands. + def create_branch(self, branch_name): + subprocess.check_output( + ['git', 'checkout', '-b', branch_name], + cwd=self.dirname) + + def show_diff(self, filename): + subprocess.call( + ['git', '--no-pager', 'diff', filename], + cwd=self.dirname) + + def has_branches(self, branches): + pass + + def add_to_git_remotes(self, remote_name, url): + try: + subprocess.check_output( + ['git', 'remote', 'add', remote_name, url], + cwd=self.dirname, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as err: + if 'already exists' in err.output: + self.logger.INFO('Fork already added to remotes') + pass + else: + raise err + except Exception as err: + raise LocalRepoException(f"Failed to add to git remotes. Error: {err}") + + def git_add(self, filename): + subprocess.check_output( + ['git', 'add', filename], + cwd=self.dirname, stderr=subprocess.STDOUT) + + def git_commit(self, message): + subprocess.check_output( + ['git', 'commit', '-m', message], + cwd=self.dirname, stderr=subprocess.STDOUT) + + def git_push(self, remote_name, branch_name): + """Push local changes to a branch of fork. + """ + # On Pagure you can not immediately push to the fork. + # And there is no api call to check that the fork is ready. + # So here is a hack: try o do it at least 4 times with an interval + # in 3 minutes. Oh well. + for attempt in range(4): + try: + self.logger.debug(f'Trying to push changes to fork (Attempt {attempt})') + subprocess.check_output( + ['git', 'push', '-f', remote_name, branch_name], + cwd=self.dirname, + stderr=subprocess.STDOUT) + break + except subprocess.CalledProcessError as err: + if 'DENIED by fallthru' in err.output: + self.logger.debug('Will sleep for 3 minutes') + time.sleep(60 * 3) + except Exception as err: + raise LocalRepoException( + f"Failed to push to a fork. Error: {err}") + else: + raise LocalRepoException('Could not push to fork, it is still not available') + self.logger.debug("Successfully pushed to a fork") diff --git a/fixrequires/pagure_fork.py b/fixrequires/pagure_fork.py new file mode 100644 index 0000000..b4ed8c3 --- /dev/null +++ b/fixrequires/pagure_fork.py @@ -0,0 +1,80 @@ +""" +""" + +from libpagure import Pagure +from libpagure.exceptions import APIError + + +PAGURE_INSTANCE = 'https://src.fedoraproject.org' + + +class PagureForkException(Exception): + + """Base exception for PagureFork. + """ + + +class PagureFork(object): + + """ + """ + + def __init__(self, pagure_repo_name, pagure_token, + pagure_user, logger, instance=PAGURE_INSTANCE): + self.package_name = pagure_repo_name + self.pagure_user = pagure_user + self.logger = logger + + self.pagure = Pagure( + pagure_repository=pagure_repo_name, + instance_url=instance, + pagure_token=pagure_token + ) + + self.fork_api = f"{self.pagure.instance}/api/0/fork" + + def do_fork(self): + self.logger .debug(f"Creating fork of {self.package_name} for user {self.pagure_user}") + try: + payload = {'wait': True, 'namespace': 'rpms', 'repo': self.package_name} + response = self.pagure._call_api(self.fork_api, method='POST', data=payload) + self.logger .debug(f"Fork created: {response}") + except APIError as err: + if 'already exists' in str(err): + self.logger .info( + f'User {self.pagure_user} already has a fork of {self.package_name}') + else: + raise err + except Exception as err: + raise PagureForkException( + f"Failed to create a fork for {self.package_name}. Error: {err}") + + def get_ssh_git_url(self): + """ + """ + git_urls_api = f"{self.fork_api}/{self.pagure_user}/rpms/{self.package_name}/git/urls" + return_value = self.pagure._call_api(git_urls_api) + return return_value['urls']['ssh'] + + def create_pull_request(self, from_branch, to_branch, + title, description, + fas_user, fas_password): + """Pagure API does not allow this yet. + https://pagure.io/pagure/issue/2803 + """ + from .pagure_pr import create_pull_request + url = ( + f'{self.pagure.instance}/login/?next={self.pagure.instance}' + f'/fork/{self.pagure_user}/rpms/{self.package_name}/diff/{to_branch}..{from_branch}') + try: + create_pull_request( + url, title, description, + fas_user, fas_password) + except Exception as err: + raise PagureForkException( + f"Failed to create a PR for {self.package_name}. Error: {err}") + + def has_epel_branch(self): + request_url = f"{self.pagure.instance}/api/0/rpms/{self.pagure.repo}/git/branches" + return_value = self.pagure._call_api(request_url) + return 'el6' in return_value["branches"] or 'epel7' in return_value["branches"] diff --git a/fixrequires/pagure_pr.py b/fixrequires/pagure_pr.py new file mode 100644 index 0000000..7fbe774 --- /dev/null +++ b/fixrequires/pagure_pr.py @@ -0,0 +1,55 @@ +from selenium import webdriver + + +def create_pull_request(url, title, description, + fas_user, fas_password): + """Create a pull request from a fork to upstream. + + Only if not created yet. + Pagure API does not allow this yet, so selenium it is. + """ + + # TODO: check if the PR is not yet there, in case the script + # is being run the second time for this project. + + driver = webdriver.Firefox() + driver.get(url) + + # Sometimes when the user is "kinited", the login page does not open, + # and you go directly to the PR page. + if driver.title == 'Login': + if not fas_user or not fas_password: + raise Exception('Please provide both FAS username and password') + + login_elem = driver.find_element_by_name('login_name') + login_elem.clear() + login_elem.send_keys(fas_user) + + password_elem = driver.find_element_by_name('login_password') + password_elem.clear() + password_elem.send_keys(fas_password) + + login_button = driver.find_element_by_id('loginbutton') + login_button.click() + + pr_title = driver.find_element_by_name('title') + pr_title_value = pr_title.get_attribute('value') + + if pr_title_value != title: + # This means the PR either contains more commits or is wrong. + # Needs to be checked manually. + raise Exception( + 'Opening the PR did not go well. ' + f'The PR title: {pr_title_value}') + + pr_init_comment = driver.find_element_by_id('initial_comment') + pr_init_comment.clear() + pr_init_comment.send_keys(description) + + create_button = driver.find_element_by_xpath( + "//input[@type='submit'][@value='Create']") + # create_button.click() + import ipdb; ipdb.set_trace() + + # TODO: check the page if it was a success. + driver.close() diff --git a/qa/__init__.py b/qa/__init__.py new file mode 100644 index 0000000..0e61fc7 --- /dev/null +++ b/qa/__init__.py @@ -0,0 +1,21 @@ +""" +""" + +from .build_mock import test_in_mock +from .build_koji import test_in_koji + +# def do_tests(package_repo, mock_build=True, koji_build=True): +# """Test the modified spec file: +# - create srpm +# - run a mock build and check results +# - run a Koji build + +# Return: a link to Koji scratch build +# """ +# package_repo.create_srpm() +# # # Locate the srpm file. +# # srpm, = package_dirname.glob('*.src.rpm') + +# build_in_mock(package_dirname) +# koji_scratch_build = build_in_koji(package_dirname) +# return koji_scratch_build \ No newline at end of file diff --git a/qa/build_koji.py b/qa/build_koji.py new file mode 100644 index 0000000..14fb801 --- /dev/null +++ b/qa/build_koji.py @@ -0,0 +1,21 @@ +""" +""" + +import re + +TASK_INFO_RE = r'Task info: (https://koji.fedoraproject.org/koji/taskinfo\?taskID=\d+)\\n' + + +def test_in_koji(package_repo, logger): + logger.debug('Running a koji build') + output = package_repo.run_koji_scratch_build() + logger.debug(f'Koji scratch build completed. Output: {output}') + + if 'completed successfully' in str(output): + # Yay Koji build completed. Find a link to a task. + koji_scratch_build = re.search(TASK_INFO_RE, str(output)).group(1) + return koji_scratch_build + else: + raise Exception( + 'Seems like Koji scratch build did not complete successfully. ' + f'Output: {output}') diff --git a/qa/build_mock.py b/qa/build_mock.py new file mode 100644 index 0000000..81b09b7 --- /dev/null +++ b/qa/build_mock.py @@ -0,0 +1,41 @@ +""" +""" +import os +import pathlib +import re +import subprocess + + +def is_unversioned(name): + """Check whether unversioned python prefix is used + in the name (e.g. python-foo). + """ + if (os.path.isabs(name) or # is an executable + os.path.splitext(name)[1]): # has as extension + return False + + return ( + name.startswith('python-') or + '-python-' in name or + name.endswith('-python') or + name == 'python') + + +def check_mockbuild_result(output): + # Check that no rpms require python-smth. + result_dir_re = r'Results and/or logs in: (\S*)' + result_dir_path = re.search(result_dir_re, output.decode('utf-8')).group(1) + result_dir = pathlib.Path(result_dir_path) + result_rpms = result_dir.glob('*.rpm') + for rpm_file in result_rpms: + requires = subprocess.check_output(['rpm', '-qRp', rpm_file]) + for require in requires.split(): + if is_unversioned(str(require)): + raise Exception(f'{require} is still not versioned') + + +def test_in_mock(package_repo, logger): + logger.debug('Running mock build') + output = package_repo.run_mockbuild() + logger.debug('Mock build completed. Checking resulting rpms') + check_mockbuild_result(output) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b280d8a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +click +selenium