From 17c515ef98e0e62b97804fd12afc2e897cd53e62 Mon Sep 17 00:00:00 2001 From: Frédéric Bérat Date: Jan 31 2025 12:27:56 +0000 Subject: New glibc-backport-process script The new script allow to execute most of the steps of the backport process. User still has to resolve conflict and may need to amend the commit message and changelogs. --- diff --git a/glibc-backport-process.py b/glibc-backport-process.py new file mode 100755 index 0000000..08727a3 --- /dev/null +++ b/glibc-backport-process.py @@ -0,0 +1,310 @@ +#!/bin/python3 + +# Helps to execute the backport process +# - Creates or update the necessary repositories (glibc-upstream, glibc-patches, +# ...) +# - Get information about the list of commits to backport, either dierctly from +# user input or from a given Jira issue +# - Execute the backport script for each Jira issue +# - Bump the NVR, update the changelog in the spec file and create a new commit +# for the changes per Jira issue. + +import argparse +from jira import JIRA +from pathlib import Path +import os +import subprocess +import textwrap +import git +from git import Repo + +import argcomplete + + +def parse_args(): + """Setup argument parser""" + + parser = argparse.ArgumentParser() + + parser.add_argument( + '--verbose', + '-V', + action='store_true', + help='Set verbosity', + ) + + parser.add_argument( + '--token', + '-t', + default='', + help='''Provide authentication token for Jira''', + ) + + parser.add_argument( + '--issue', + '-i', + default='', + help='''Provide Jira issue to create the backport for. + Can be a space separated list''', + ) + + parser.add_argument( + '--commit-hash', + '-c', + default='', + help='''Provide commit hashes to create the backport for. + Can be a space separated list''', + ) + + parser.add_argument( + '--branch', + '-b', + default='c10s', + help='''The centos branch to checkout, default c10s''', + ) + + parser.add_argument( + '--pristine', + '-p', + default=f'{os.getcwd()}/glibc-pristine', + help=f'''Path to glibc pristine sources. + Defaults {os.getcwd()}/glibc-pristine''', + ) + + parser.add_argument( + '--maintainers', + '-m', + default=f'{os.getcwd()}/glibc-maintainer-scripts', + help=f'''Path to glibc maintainers script sources. + Defaults {os.getcwd()}/glibc-maintainer-scripts''', + ) + + argcomplete.autocomplete(parser) + + return parser.parse_args() + + +def find_hashes(jira, key): + issue = jira.issue(key) + assert issue + + hashes = list() + + if issue.fields.customfield_12324041: + hashes = [{'hash': h} for h in issue.fields.customfield_12324041.split()] + + if hashes: + return hashes + + description = issue.fields.description + + for line in description.splitlines(): + if not "commit " in line: + continue + splitted = line.strip('{}').split() + candidate = splitted[splitted.index('commit') + 1] + + if len(candidate) == 40: + hashes.append({'hash': candidate}) + + if hashes: + return hashes + + comments = { comment for comment in issue.fields.comment.comments } + + for comment in comments: + for line in comment.body.splitlines(): + if not "commit " in line: + continue + splitted = line.strip('{}').split() + candidate = splitted[splitted.index('commit') + 1] + + if len(candidate) == 40: + hashes.append({'hash': candidate}) + + return hashes + + +def list_setup(jira, base_list, r_pristine, commit_hashes): + issues_simplified = list() + + for issue in sorted(base_list): + hashes = [{'hash':h} for h in commit_hashes] or find_hashes(jira, issue) + + if not hashes: + print(f'No commit hashes found for {issue.key}') + continue + + titles = list() + for h in hashes: + c = r_pristine.commit(h['hash']) + h['title'] = c.summary + h['count'] = c.count() + + issues_simplified.append({'key': issue, + 'hashes': sorted(hashes, key=lambda x: x['count'])}) + + return issues_simplified + + +def git_setup(branch, pristine_path, maintainer_scripts): + """Check that we are in centos git repository. + Setup glibc-pristine and glibc-patches git repositories if they don't + exist. + """ + r_glibc = Repo(".") + assert r_glibc + assert "redhat/centos-stream" in r_glibc.remote().url + assert not r_glibc.is_dirty() + + r_glibc.git.checkout(branch) + #r_glibc.git.pull() + + try: + r_pristine = Repo(pristine_path) + except git.exc.NoSuchPathError: + url = "https://sourceware.org/git/glibc" + print(f'Cloning {url} into {pristine_path}') + r_pristine = Repo.clone_from(url, pristine_path) + assert r_pristine + assert not r_pristine.is_dirty() + r_pristine.git.checkout('master') + r_pristine.git.pull() + + try: + r_maint = Repo(maintainer_scripts) + except git.exc.NoSuchPathError: + url = "https://pagure.io/glibc-maintainer-scripts" + print(f'Cloning {url} into {maintainer_scripts}') + r_maint = Repo.clone_from(url, maintainer_scripts) + assert r_maint + assert not r_maint.is_dirty() + r_maint.git.checkout('master') + r_maint.git.pull() + + command = f'{maintainer_scripts}/glibc-patches-to-git.py --verbose --branch {branch}' + print(f"Executing: {command}") + ret = subprocess.check_call(command.split()) + print(f"Command returned {ret}") + assert not ret + + r_patches = Repo("./glibc-patches") + assert r_patches + r_patches.git.fetch([pristine_path, 'origin/master:upstream-master'], + '--no-tags') + + return r_glibc, r_pristine + + +def main_loop(r_glibc, m_scripts, issue_sorted): + prev_line = "PLACEHOLDER" + + command = f"rpmdev-bumpspec -c {prev_line} glibc.spec" + print(f"Bumping spec with {prev_line}") + ret = subprocess.check_call(command.split()) + + for issue in issue_sorted: + print(f"Working on {issue['key']} ({issue_sorted.index(issue) + 1}/{len(issue_sorted)}):") + if len(issue['hashes']) > 1: + pattern = f'glibc-{issue["key"]}-1.patch' + else: + pattern = f'glibc-{issue["key"]}.patch' + + command = f"{m_scripts}/glibc-backport-patch.py --name {pattern}" + for commit in issue['hashes']: + command = f'{command} {commit["hash"]}' + + print(f"Executing: {command}") + ret = subprocess.run(command.split(), cwd=f"{r_glibc.working_tree_dir}/glibc-patches") + print(f"Command returned {ret}") + if ret.returncode: + print() + print("Please fix cherry-pick conflicts before proceeding...") + input() + + command = f"{m_scripts}/glibc-git-to-patches.py" + subprocess.check_call(command) + + key = issue['key'] + title = f'Backport: {issue["hashes"][-1]["title"]}' + + lines = [] + for h in issue['hashes']: + lines.extend(textwrap.wrap(f'- Backport: {h["title"]} ({key})')) + + changelog = "" + for new_line in lines: + if "PLACEHOLDER" in prev_line: + command = [f'sed', + '-i', + f's/- PLACEHOLDER/{new_line}/', + 'glibc.spec'] + changelog = f"{new_line}" + else: + if new_line[0] == '-': + command = [f'sed', + '-i', + f'/{prev_line}/a {new_line}', + 'glibc.spec'] + changelog = f"{changelog}\n{new_line}" + else: + command = [f'sed', + '-i', + f'/{prev_line}/a \\ \\ {new_line}', + 'glibc.spec'] + changelog = f"{changelog}\n {new_line}" + + subprocess.check_call(command) + prev_line = new_line + + r_glibc.git.add(f'glibc-{key}*.patch') + r_glibc.git.add(f'glibc.spec') + + message = f"{title}\n\n{changelog}\n\nResolves: {key}" + + r_glibc.index.commit(message) + + +def main(): + """Main function + + Setup argument parser, initialize default configuration, validates user + input and perform the backport. + """ + args = parse_args() + + base_list = args.issue.split() + + commit_hashes = args.commit_hash.split() + + if not base_list: + printf("No issue provided") + exit(1) + + if all([len(base_list) > 1, len(commit_hashes) > 0]): + printf(f"Only one issue can be provided is commit hashes are given.") + exit(1) + + if all([not len(commit_hashes), not args.token]): + print(f'Authentication token must be given ({args.token}) when no hashes are provided.') + exit(1) + + jira = None + if args.token: + jira = JIRA(server="https://issues.redhat.com", token_auth=args.token) + + if not jira: + printf(f'Failed to access jira instance.') + exit(1) + + r_glibc, r_pristine = git_setup(args.branch, args.pristine, args.maintainers) + + issue_list = list_setup(jira, base_list, r_pristine, commit_hashes) + + main_loop(r_glibc, + args.maintainers, + sorted(issue_list, key=lambda x: x['hashes'][0]['count'])) + + +if __name__ == '__main__': + main()