From 5fe352684d32edb90ba33d2791cb8098717850f6 Mon Sep 17 00:00:00 2001 From: Hunor Csomortáni Date: Jun 25 2021 16:28:09 +0000 Subject: Refactor 'source-git init' Signed-off-by: Hunor Csomortáni --- diff --git a/packit/api.py b/packit/api.py index 7566c60..46777fe 100644 --- a/packit/api.py +++ b/packit/api.py @@ -13,6 +13,8 @@ from datetime import datetime from pathlib import Path from typing import Sequence, Callable, List, Tuple, Dict, Iterable, Optional, Union +import git + from ogr.abstract import PullRequest from pkg_resources import get_distribution, DistributionNotFound from tabulate import tabulate @@ -987,40 +989,39 @@ class PackitAPI: v = PackageConfigValidator(config_path, config_content) return v.validate() - def create_sourcegit_from_upstream( + def init_source_git( self, + dist_git: git.Repo, + source_git: git.Repo, + upstream_ref: str, upstream_url: Optional[str] = None, - upstream_ref: Optional[str] = None, - dist_git_path: Optional[Path] = None, - dist_git_branch: Optional[str] = None, - fedora_package: Optional[str] = None, - centos_package: Optional[str] = None, + upstream_remote: Optional[str] = None, pkg_tool: Optional[str] = None, pkg_name: Optional[str] = None, ): """ - generate a source-git repo from provided upstream repo - and a corresponding package in Fedora/CentOS ecosystem - - :param upstream_url: upstream repo URL we want to use as a base - :param upstream_ref: upstream git-ref to use as a base - :param fedora_package: pick up specfile and downstream sources from this fedora package - :param centos_package: pick up specfile and downstream sources from this centos package - :param dist_git_branch: branch in dist-git to use - :param dist_git_path: path to a local clone of a dist-git repo - :param pkg_tool: name or path of the packaging tool executable to be used to interact - with the lookaside cache - :param pkg_name: name of the package in the distro + Initialize a source-git repo from dist-git, that is: add configuration, packaging files + needed by the distribution and transform the distribution patches into Git commits. + + Args: + dist_git: Dist-git repository to be used for initialization. + source_git: Git repository to be initialized as a source-git repo. + upstream_ref: Upstream ref which is going to be the starting point of the + source-git history. This can be a branch, tag or commit sha. It is expected + that the current HEAD and this ref point to the same commit. + upstream_url: Git URL to be saved in the source-git configuration. + upstream_remote: Name of the remote from which the fetch URL is taken as the Git URL + of the upstream project to be saved in the source-git configuration. + pkg_tool: Packaging tool to be used to interact with the dist-git repo. + pkg_name: Name of the package in dist-git. """ sgg = SourceGitGenerator( config=self.config, - local_project=self.upstream_local_project, - upstream_url=upstream_url, + dist_git=dist_git, + source_git=source_git, upstream_ref=upstream_ref, - dist_git_path=dist_git_path, - dist_git_branch=dist_git_branch, - fedora_package=fedora_package, - centos_package=centos_package, + upstream_url=upstream_url, + upstream_remote=upstream_remote, pkg_tool=pkg_tool, pkg_name=pkg_name, ) diff --git a/packit/cli/source_git_init.py b/packit/cli/source_git_init.py index dbb2e52..43326b7 100644 --- a/packit/cli/source_git_init.py +++ b/packit/cli/source_git_init.py @@ -4,89 +4,94 @@ """Command to initialize a source-git repository""" import logging -import os -import pathlib from typing import Optional import click +import git -from packit.cli.types import LocalProjectParameter -from packit.cli.utils import get_packit_api +from packit.cli.types import GitRepoParameter +from packit.api import PackitAPI from packit.config.config import pass_config from packit.config import get_context_settings -from packit.exceptions import PackitNotAGitRepoException logger = logging.getLogger(__name__) @click.command("init", context_settings=get_context_settings()) -@click.argument("path_or_url", type=LocalProjectParameter(), default=os.path.curdir) +@click.argument("upstream_ref") +@click.argument("source_git", type=GitRepoParameter(from_ref_param="upstream_ref")) +@click.argument("dist_git", type=GitRepoParameter()) @click.option( "--upstream-url", - help="URL or local path to the upstream project; " - "defaults to current git repository", + help="""Git URL of the upstream repository. It is saved + in the source-git configuration if it is specified.""", ) @click.option( - "--upstream-ref", - help="Use this upstream git ref as a base for your source-git repo; " - "defaults to current tip of the git repository", + "--upstream-remote", + help="""Name of the remote pointing to the upstream repository. + If --upstream-url is not specified, the fetch URL of this remote + is saved in the source-git configuration as the Git URL of the + upstream project. Defaults to 'origin'.""", ) @click.option( - "--dist-git-branch", - help="Get spec file from this downstream branch, " - "for Fedora this defaults to main, for CentOS it's c9s. " - "When --dist-git-path is set, the default is the branch which is already checked out.", -) -@click.option( - "--dist-git-path", - help="Path to the dist-git repo to use. If this is defined, " - "--fedora-package and --centos-package are ignored.", + "--pkg-tool", + help="""Name or path of the packaging tool used to work + with sources in the dist-git repo. A variant of 'rpkg'. + Defaults to 'fedpkg' or the tool configured in the Packit + configuration.""", ) @click.option( - "--pkg-tool", - help="Name or path of the packaging tool used to work " - "with sources in the dist-git repo. A variant of 'rpkg'.", + "--pkg-name", + help="""The name of the package in the distro. + Defaults to the directory name of DIST_GIT.""", ) -@click.option("--pkg-name", help="The name of the package in the distro") @pass_config def source_git_init( config, - path_or_url, - upstream_url, - upstream_ref, - dist_git_branch, - dist_git_path: Optional[str], + dist_git: git.Repo, + source_git: git.Repo, + upstream_ref: str, + upstream_url: Optional[str], + upstream_remote, pkg_tool: Optional[str], pkg_name: Optional[str], ): - """Initialize a source-git repository + """Initialize SOURCE_GIT as a source-git repo by applying downstream + patches from DIST_GIT as Git commits on top of UPSTREAM_REF. + + UPSTREAM_REF is a tag, branch or commit from SOURCE_GIT. + + SOURCE_GIT and DIST_GIT are paths to the source-git and dist-git + repos. Branch names can be specified, separated by colons. + + If a branch name is specified for SOURCE_GIT, the branch is checked + out and reset to UPSTREAM_REF. + + If a branch name is specified for DIST_GIT, the branch is checked + out before setting up the source-git repo. This branch is expected + to exist. To learn more about source-git, please check https://packit.dev/docs/source-git/ + + Examples: + + \b + $ packit source-git init v2.3.1 src/acl:rawhide rpms/acl:rawhide + $ packit source-git init --pkg-tool centpkg v2.3.1 src/acl rpms/acl """ logger.warning( "Generating source-git repositories is experimental, " "please give us feedback if it does things differently than you expect." ) - try: - api = get_packit_api( - config=config, local_project=path_or_url, load_packit_yaml=False - ) - except PackitNotAGitRepoException: - logger.error( - "The init command is expected to be run in a git repository. " - "Current branch in the repo will be turned into a source-git repo. " - "We suggest to run the command " - "in a blank git repository or in a new branch of the upstream project." - ) - raise - dg_path = pathlib.Path(dist_git_path) if dist_git_path else None - api.create_sourcegit_from_upstream( - upstream_url=upstream_url, + api = PackitAPI(config=config, package_config=None) + api.init_source_git( + dist_git=dist_git, + source_git=source_git, upstream_ref=upstream_ref, - dist_git_path=dg_path, - dist_git_branch=dist_git_branch, + upstream_url=upstream_url, + upstream_remote=upstream_remote, pkg_tool=pkg_tool or config.pkg_tool, pkg_name=pkg_name, ) diff --git a/packit/cli/types.py b/packit/cli/types.py index 91d7786..8931c13 100644 --- a/packit/cli/types.py +++ b/packit/cli/types.py @@ -1,29 +1,12 @@ -# MIT License -# -# Copyright (c) 2019 Red Hat, Inc. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT import logging import os +from typing import Optional import click +import git from packit.local_project import LocalProject from packit.utils.repo import git_remote_url_to_https_url @@ -86,3 +69,63 @@ class LocalProjectParameter(click.ParamType): return local_project except Exception as ex: self.fail(ex, param, ctx) + + +class GitRepoParameter(click.ParamType): + """Parameter type to represent a Git repository on the local disk, and an + optional branch, in the format :. + + Attributes: + from_ref_param: Name of the CLI parameter which tells the start point of the branch + to be created, if the branch doesn't exist yet. + """ + + name = "git_repo" + + def __init__(self, from_ref_param: Optional[str] = None): + super().__init__() + self.from_ref_param = from_ref_param + + def convert(self, value, param, ctx) -> git.Repo: + if isinstance(value, git.Repo): + return value + if not isinstance(value, str): + self.fail(f"{value!r} is not a string") + + try: + path, _, branch = value.partition(":") + repo = git.Repo(path) + if not branch: + return repo + + branch_exists = True + try: + repo.rev_parse(branch) + except git.BadName: + branch_exists = False + + if self.from_ref_param is not None: + if ctx.params.get(self.from_ref_param): + repo.git.checkout("-B", branch, ctx.params[self.from_ref_param]) + else: + self.fail( + f"Unable to create branch {branch!r} because " + f"{self.from_ref_param!r} is not specified", + param, + ctx, + ) + elif branch_exists: + repo.git.checkout(branch) + else: + self.fail( + f"Cannot check out branch {branch!r} because it does not exist", + param, + ctx, + ) + + return repo + + except git.NoSuchPathError: + self.fail(f"{path!r} does not exist", param, ctx) + except git.GitCommandError as ex: + self.fail(ex, param, ctx) diff --git a/packit/constants.py b/packit/constants.py index 9d42203..83b8f38 100644 --- a/packit/constants.py +++ b/packit/constants.py @@ -1,24 +1,5 @@ -# MIT License -# -# Copyright (c) 2019 Red Hat, Inc. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT DG_PR_COMMENT_KEY_SG_PR = "Source-git pull request ID" DG_PR_COMMENT_KEY_SG_COMMIT = "Source-git commit" @@ -39,12 +20,6 @@ DEFAULT_ARCHIVE_EXT = ".tar.gz" DEFAULT_BODHI_NOTE = "New upstream release: {version}" FEDORA_DOMAIN = "fedoraproject.org" -CENTOS_DOMAIN = "centos.org" -CENTOS_STREAM_GITLAB_DOMAIN = "gitlab.com" -CENTOS_STREAM_GITLAB_NAMESPACE = "redhat/centos-stream" -CENTOS_STREAM_GITLAB = ( - CENTOS_STREAM_GITLAB_DOMAIN + "/" + CENTOS_STREAM_GITLAB_NAMESPACE -) PROD_DISTGIT_HOSTNAME = f"src.{FEDORA_DOMAIN}" PROD_DISTGIT_URL = f"https://{PROD_DISTGIT_HOSTNAME}/" @@ -135,7 +110,7 @@ RPM_MACROS_FOR_PREP = [ "patch_name=`basename %{1}` && " "commit_msg=`%{__git} log --format=%B -n1` && " r'metadata_commit_msg=`printf "patch_name: $patch_name\\n' - r'present_in_specfile: true\\nsquash_commits: true"` && ' + r'squash_commits: true"` && ' '%{__git} commit --amend -m "$commit_msg" -m "$metadata_commit_msg"', # do the same of %autosetup -Sgit # that is, apply packit patch metadata to the patch commit @@ -144,6 +119,6 @@ RPM_MACROS_FOR_PREP = [ "__scm_apply_git(qp:m:) " "%{__git} apply --index %{-p:-p%{-p*}} - && " "patch_name=`basename %{1}` && " - r'metadata_commit_msg=`printf "patch_name: $patch_name\\npresent_in_specfile: true\\n"` && ' + r'metadata_commit_msg=`printf "patch_name: $patch_name\\n"` && ' '%{__git} commit %{-q} -m %{-m*} -m "${metadata_commit_msg}" --author "%{__scm_author}"', ] diff --git a/packit/source_git.py b/packit/source_git.py index 799b427..7e5960f 100644 --- a/packit/source_git.py +++ b/packit/source_git.py @@ -1,42 +1,29 @@ # Copyright Contributors to the Packit project. # SPDX-License-Identifier: MIT + """ packit started as source-git and we're making a source-git module after such a long time, weird """ + import configparser import logging import shutil -import tarfile -import tempfile import textwrap from pathlib import Path -from typing import Optional, Tuple, List, Dict +from typing import Optional, List, Dict +import git import yaml -from git import GitCommandError -from ogr.parsing import parse_git_repo from rebasehelper.exceptions import LookasideCacheError from rebasehelper.helpers.lookaside_cache_helper import LookasideCacheHelper -from packit.config.common_package_config import CommonPackageConfig -from packit.config.config import Config -from packit.config.package_config import PackageConfig -from packit.constants import ( - RPM_MACROS_FOR_PREP, - FEDORA_DOMAIN, - CENTOS_DOMAIN, - CENTOS_STREAM_GITLAB, - CENTOS_STREAM_GITLAB_DOMAIN, - CENTOS_STREAM_GITLAB_NAMESPACE, -) -from packit.distgit import DistGit +from packit.config import Config +from packit.constants import RPM_MACROS_FOR_PREP from packit.exceptions import PackitException -from packit.local_project import LocalProject from packit.patches import PatchMetadata +from packit.pkgtool import PkgTool from packit.utils import ( run_command, - clone_centos_8_package, - clone_centos_9_package, get_default_branch, ) from packit.specfile import Specfile @@ -44,105 +31,6 @@ from packit.specfile import Specfile logger = logging.getLogger(__name__) -class CentOS8DistGit(DistGit): - """ - CentOS dist-git layout implementation for 8: CentOS Linux 8 and CentOS Stream 8 - which lives in git.centos.org - """ - - # spec files are stored in this dir in dist-git - spec_dir_name = "SPECS" - - # sources are stored in this dir in dist-git - # this applies to CentOS Stream 8 and CentOS Linux 7 and 8 - source_dir_name = "SOURCES" - - @classmethod - def clone( - cls, - config: Config, - package_config: CommonPackageConfig, - path: Path, - branch: str = None, - ) -> "CentOS8DistGit": - clone_centos_8_package( - package_config.downstream_package_name, path, branch=branch - ) - lp = LocalProject(working_dir=path) - return cls(config, package_config, local_project=lp) - - -class CentOS9DistGit(DistGit): - """ - CentOS dist-git layout implementation for CentOS Stream 9 - which lives in gitlab.com/redhat/centos-stream/rpms - """ - - # spec files are stored in this dir in dist-git - spec_dir_name = "" - - # sources are stored in this dir in dist-git - source_dir_name = "" - - @classmethod - def clone( - cls, - config: Config, - package_config: CommonPackageConfig, - path: Path, - branch: str = None, - ) -> "CentOS9DistGit": - clone_centos_9_package( - package_config.downstream_package_name, path, branch=branch - ) - lp = LocalProject(working_dir=path) - return cls(config, package_config, local_project=lp) - - -def get_distgit_kls_from_repo( - repo_path: Path, config: Config -) -> Tuple[DistGit, Optional[str], Optional[str]]: - """ - :return: DistGit instance, centos package name, fedora package name - """ - path = Path(repo_path) - pc = PackageConfig(downstream_package_name=path.name) - lp = LocalProject(working_dir=path) - - lp_git_repo = parse_git_repo(lp.git_url) - - if FEDORA_DOMAIN in lp_git_repo.hostname: - return DistGit(config, pc, local_project=lp), None, path.name - elif CENTOS_DOMAIN in lp_git_repo.hostname: - return CentOS8DistGit(config, pc, local_project=lp), path.name, None - elif ( - CENTOS_STREAM_GITLAB_DOMAIN == lp_git_repo.hostname - and lp_git_repo.namespace.find(CENTOS_STREAM_GITLAB_NAMESPACE) == 0 - ): - return CentOS9DistGit(config, pc, local_project=lp), path.name, None - raise PackitException( - f"Dist-git URL {lp.git_url} not recognized, we expected one of: " - f"{FEDORA_DOMAIN}, {CENTOS_DOMAIN} or {CENTOS_STREAM_GITLAB}" - ) - - -def get_tarball_comment(tarball_path: str) -> Optional[str]: - """Return the comment header for the tarball - - If written by git-archive, this contains the Git commit ID. - Return None if the file is invalid or does not contain a comment. - - shamelessly stolen: - https://pagure.io/glibc-maintainer-scripts/blob/master/f/glibc-sync-upstream.py#_75 - """ - try: - with tarfile.open(tarball_path) as tar: - return tar.pax_headers["comment"] - except Exception as ex: - logger.debug(f"Could not get 'comment' header from the tarball: {ex}") - return None - - # https://stackoverflow.com/questions/13518819/avoid-references-in-pyyaml # mypy hated the suggestion from the SA ^, hence an override like this class SafeDumperWithoutAliases(yaml.SafeDumper): @@ -153,8 +41,23 @@ class SafeDumperWithoutAliases(yaml.SafeDumper): class SourceGitGenerator: """ - generate a source-git repo from provided upstream repo - and a corresponding package in Fedora/CentOS ecosystem + Set up a source-git repo in an upstream repo taking downstream patches + from the corresponding package in Fedora/CentOS/RHEL. + + Attributes: + config: Packit configuration. + source_git: Git repository to be initialized as a source-git repo. + dist_git: Dist-git repository to be used for initialization. + upstream_ref: Upstream ref which is going to be the starting point of the + source-git history. This can be a branch, tag or commit sha. It is expected + that the current HEAD and this ref point to the same commit. + upstream_url: Git URL to be saved in the source-git configuration. If not specified, + the fetch URL of the 'origin' remote us used, unless 'upstream_remote' is set. + upstream_remote: Name of the remote from which the fetch URL is taken as the Git URL + of the upstream project to be saved in the source-git configuration. + pkg_tool: Packaging tool to be used to interact with the dist-git repo. + pkg_name: Name of the package in dist-git. + distr_dir: The path to the .distro directory in source-git created during set up. """ # we store downstream content in source-git in this subdir @@ -162,201 +65,38 @@ class SourceGitGenerator: def __init__( self, - local_project: LocalProject, config: Config, + source_git: git.Repo, + dist_git: git.Repo, + upstream_ref: str, upstream_url: Optional[str] = None, - upstream_ref: Optional[str] = None, - dist_git_path: Optional[Path] = None, - dist_git_branch: Optional[str] = None, - fedora_package: Optional[str] = None, - centos_package: Optional[str] = None, + upstream_remote: Optional[str] = None, pkg_tool: Optional[str] = None, pkg_name: Optional[str] = None, - tmpdir: Optional[Path] = None, ): - """ - :param local_project: this source-git repo - :param config: global configuration - :param upstream_url: upstream repo URL we want to use as a base - :param upstream_ref: upstream git-ref to use as a base - :param dist_git_path: path to a local clone of a dist-git repo - :param dist_git_branch: branch in dist-git to use - :param fedora_package: pick up specfile and downstream sources from this fedora package - :param centos_package: pick up specfile and downstream sources from this centos package - :param pkg_tool: name or path of the packaging tool executable to be used to interact - with the lookaside cache - :param pkg_name: name of the package in the distro - :param tmpdir: path to a directory where temporary repos (upstream, - dist-git) will be cloned - """ - self.local_project = local_project self.config = config - self.tmpdir = tmpdir or Path(tempfile.mkdtemp(prefix="packit-sg-")) - self._dist_git: Optional[DistGit] = None - self._primary_archive: Optional[Path] = None - self._upstream_ref: Optional[str] = upstream_ref - self.dist_git_branch = dist_git_branch - self.distro_dir = Path(self.local_project.working_dir, self.DISTRO_DIR) - self.pkg_tool = pkg_tool - self.pkg_name = pkg_name - self._patch_comments: dict = {} - - logger.info( - f"The source-git repo is going to be created in {local_project.working_dir}." + self.source_git = source_git + self.dist_git = dist_git + self.upstream_ref = upstream_ref + self.upstream_url = upstream_url or next( + self.source_git.remote(upstream_remote or "origin").urls ) + self.pkg_tool = pkg_tool + self.pkg_name = pkg_name or Path(self.dist_git.working_dir).name - if dist_git_path: - ( - self._dist_git, - self.centos_package, - self.fedora_package, - ) = get_distgit_kls_from_repo(dist_git_path, config) - self.dist_git_path = dist_git_path - self.package_config = self.dist_git.package_config - else: - self.centos_package = centos_package - self.fedora_package = fedora_package - if centos_package: - self.package_config = PackageConfig( - downstream_package_name=centos_package - ) - elif fedora_package: - self.fedora_package = ( - self.fedora_package or local_project.working_dir.name - ) - self.package_config = PackageConfig( - downstream_package_name=fedora_package - ) - else: - raise PackitException( - "Please tell us the name of the package in the downstream." - ) - self.dist_git_path = self.tmpdir.joinpath( - self.package_config.downstream_package_name - ) - - if upstream_url: - if Path(upstream_url).is_dir(): - self.upstream_repo_path: Path = Path(upstream_url) - self.upstream_lp: LocalProject = LocalProject( - working_dir=self.upstream_repo_path - ) - else: - self.upstream_repo_path = self.tmpdir.joinpath( - f"{self.package_config.downstream_package_name}-upstream" - ) - self.upstream_lp = LocalProject( - git_url=upstream_url, working_dir=self.upstream_repo_path - ) - else: - # $CWD is the upstream repo and we just need to pick - # downstream stuff - self.upstream_repo_path = self.local_project.working_dir - self.upstream_lp = self.local_project - - @property - def primary_archive(self) -> Path: - if not self._primary_archive: - self._primary_archive = self.dist_git.download_upstream_archive() - return self._primary_archive - - @property - def dist_git(self) -> DistGit: - if not self._dist_git: - self._dist_git = self._get_dist_git() - # we need to parse the spec twice - # https://github.com/rebase-helper/rebase-helper/issues/848 - self._dist_git.specfile.reload() - return self._dist_git + self._dist_git_specfile: Optional[Specfile] = None + self.distro_dir = Path(self.source_git.working_dir, self.DISTRO_DIR) + self._dist_git: Optional[git.Repo] = None + self._patch_comments: dict = {} @property - def upstream_ref(self) -> str: - if self._upstream_ref is None: - self._upstream_ref = get_tarball_comment(str(self.primary_archive)) - if self._upstream_ref: - logger.info( - "upstream base ref was not set, " - f"discovered it from the archive: {self._upstream_ref}" - ) - else: - # fallback to HEAD - try: - self._upstream_ref = self.local_project.commit_hexsha - except ValueError as ex: - raise PackitException( - "Current branch seems to be empty - we cannot get the hash of " - "the top commit. We need to set upstream_ref in packit.yaml to " - "distinct between upstream and downstream changes. " - "Please set --upstream-ref or pull the upstream git history yourself. " - f"Error: {ex}" - ) - logger.info( - "upstream base ref was not set, " - f"falling back to the HEAD commit: {self._upstream_ref}" - ) - return self._upstream_ref - - def _get_dist_git( - self, - ) -> DistGit: - """ - For given package names, clone the dist-git repo in the given directory - and return the DistGit class - - :return: DistGit instance - """ - if self.centos_package: - self.dist_git_branch = self.dist_git_branch or "c9s" - # let's be sure to cover anything 9 related, - # even though "c9" will probably never be a thing - if "c9" in self.dist_git_branch: - return CentOS9DistGit.clone( - config=self.config, - package_config=self.package_config, - path=self.dist_git_path, - branch=self.dist_git_branch, - ) - return CentOS8DistGit.clone( - config=self.config, - package_config=self.package_config, - path=self.dist_git_path, - branch=self.dist_git_branch, - ) - else: - # If self.dist_git_branch is None we will checkout/store repo's default branch - dg = DistGit.clone( - config=self.config, - package_config=self.package_config, - path=self.dist_git_path, - branch=self.dist_git_branch, + def dist_git_specfile(self) -> Specfile: + if not self._dist_git_specfile: + path = str(Path(self.dist_git.working_dir, f"{self.pkg_name}.spec")) + self._dist_git_specfile = Specfile( + path, sources_dir=self.dist_git.working_dir ) - self.dist_git_branch = ( - self.dist_git_branch or dg.local_project.git_project.default_branch - ) - return dg - - def _pull_upstream_ref(self): - """ - Pull the base ref from upstream to our source-git repo - """ - # fetch operation is pretty intense - # if upstream_ref is a commit, we need to fetch everything - # if it's a tag or branch, we can only fetch that ref - self.local_project.fetch( - str(self.upstream_lp.working_dir), "+refs/heads/*:refs/remotes/origin/*" - ) - self.local_project.fetch( - str(self.upstream_lp.working_dir), - "+refs/remotes/origin/*:refs/remotes/origin/*", - ) - try: - next(self.local_project.get_commits()) - except GitCommandError as ex: - logger.debug(f"Can't get next commit: {ex}") - # the repo is empty, rebase would fail - self.local_project.reset(self.upstream_ref) - else: - self.local_project.rebase(self.upstream_ref) + return self._dist_git_specfile def _run_prep(self): """ @@ -370,29 +110,27 @@ class SourceGitGenerator: 'by running `rpmbuild -bp` but we cannot find "_packitpatch" command on PATH: ' "please install packit as an RPM." ) - logger.info( - f"expanding %prep section in {self.dist_git.local_project.working_dir}" - ) + logger.info(f"expanding %prep section in {self.dist_git.working_dir}") rpmbuild_args = [ "rpmbuild", "--nodeps", "--define", - f"_topdir {str(self.dist_git.local_project.working_dir)}", + f"_topdir {str(self.dist_git.working_dir)}", "-bp", "--define", - f"_specdir {str(self.dist_git.absolute_specfile_dir)}", + f"_specdir {str(self.dist_git.working_dir)}", "--define", - f"_sourcedir {str(self.dist_git.absolute_source_dir)}", + f"_sourcedir {str(self.dist_git.working_dir)}", ] rpmbuild_args += RPM_MACROS_FOR_PREP if logger.level <= logging.DEBUG: # -vv can be super-duper verbose rpmbuild_args.append("-v") - rpmbuild_args.append(str(self.dist_git.absolute_specfile_path)) + rpmbuild_args.append(self.dist_git_specfile.path) run_command( rpmbuild_args, - cwd=self.dist_git.local_project.working_dir, + cwd=self.dist_git.working_dir, print_live=True, ) @@ -401,15 +139,15 @@ class SourceGitGenerator: Read "sources" file from the dist-git repo and return a list of dicts with path and url to sources stored in the lookaside cache """ - pkg_tool = "centpkg" if self.centos_package else "fedpkg" + pkg_tool = self.pkg_tool or self.config.pkg_tool try: config = LookasideCacheHelper._read_config(pkg_tool) base_url = config["lookaside"] except (configparser.Error, KeyError) as e: raise LookasideCacheError("Failed to read rpkg configuration") from e - package = self.dist_git.package_config.downstream_package_name - basepath = self.dist_git.local_project.working_dir + package = self.pkg_name + basepath = self.dist_git.working_dir sources = [] for source in LookasideCacheHelper._read_sources(basepath): @@ -427,7 +165,7 @@ class SourceGitGenerator: return sources def get_BUILD_dir(self): - path = self.dist_git.local_project.working_dir + path = Path(self.dist_git.working_dir) build_dirs = [d for d in (path / "BUILD").iterdir() if d.is_dir()] if len(build_dirs) > 1: raise RuntimeError(f"More than one directory found in {path / 'BUILD'}") @@ -435,34 +173,32 @@ class SourceGitGenerator: raise RuntimeError(f"No subdirectory found in {path / 'BUILD'}") return build_dirs[0] - def _rebase_patches(self, from_branch: str, patch_comments: Dict[str, List[str]]): + def _rebase_patches(self, patch_comments: Dict[str, List[str]]): """Rebase current branch against the from_branch Args: - from_branch: Branch in the BUILD directory from which downstream - commits are fetched to the source-git repository. patch_comments: dict to map patch names to comment lines serving as a description of those patches. """ to_branch = "dist-git-commits" # temporary branch to store the dist-git history - logger.info(f"Rebase patches from dist-git {from_branch}.") BUILD_dir = self.get_BUILD_dir() - self.local_project.fetch(BUILD_dir, f"+{from_branch}:{to_branch}") + prep_repo = git.Repo(BUILD_dir) + from_branch = get_default_branch(prep_repo) + logger.info(f"Rebase patches from dist-git {from_branch}.") + self.source_git.git.fetch(BUILD_dir, f"+{from_branch}:{to_branch}") # transform into {patch_name: patch_id} patch_ids = { p.get_patch_name(): p.index - for p in self.dist_git.specfile.patches.get("applied", []) + for p in self.dist_git_specfile.patches.get("applied", []) } # -2 - drop first commit which represents tarball unpacking # -1 - reverse order, HEAD is last in the sequence - patch_commits = list( - LocalProject(working_dir=BUILD_dir).get_commits(from_branch) - )[-2::-1] + patch_commits = list(prep_repo.iter_commits(from_branch))[-2::-1] for commit in patch_commits: - self.local_project.git_repo.git.cherry_pick( + self.source_git.git.cherry_pick( commit.hexsha, keep_redundant_commits=True, allow_empty=True, @@ -479,9 +215,9 @@ class SourceGitGenerator: message += "description: |-\n" for line in patch_comments.get(metadata.name, []): message += f" {line}\n" - self.local_project.commit(message, amend=True) + self.source_git.git.commit(message=message, amend=True) - self.local_project.git_repo.git.branch("-D", to_branch) + self.source_git.git.branch("-D", to_branch) def _populate_distro_dir(self): """Copy files used in the distro to package and test the software to .distro. @@ -490,15 +226,16 @@ class SourceGitGenerator: PackitException, if the dist-git repository is not pristine, that is, there are changed or untracked files in it. """ - dist_git_repo = self.dist_git.local_project.git_repo dist_git_is_pristine = ( - not dist_git_repo.git.diff() and not dist_git_repo.git.clean("-xdn") + not self.dist_git.git.diff() and not self.dist_git.git.clean("-xdn") ) if not dist_git_is_pristine: raise PackitException( "Cannot initialize a source-git repo. " "The corresponding dist-git repository at " - f"{dist_git_repo.working_dir} is not pristine." + f"{self.dist_git.working_dir!r} is not pristine." + "Use 'git reset --hard HEAD' to reset changed files and " + "'git clean -xdff' to delete untracked files and directories." ) command = ["rsync", "--archive", "--delete"] @@ -506,7 +243,7 @@ class SourceGitGenerator: command += ["--filter", f"exclude {exclude}"] command += [ - str(self.dist_git.local_project.working_dir) + "/", + str(self.dist_git.working_dir) + "/", str(self.distro_dir), ] @@ -525,15 +262,14 @@ class SourceGitGenerator: def _configure_syncing(self): """Populate source-git.yaml""" package_config = {} - if self.upstream_lp.git_url: - package_config["upstream_project_url"] = self.upstream_lp.git_url package_config.update( { + "upstream_project_url": self.upstream_url, "upstream_ref": self.upstream_ref, "downstream_package_name": self.pkg_name, "specfile_path": f"{self.DISTRO_DIR}/{self.pkg_name}.spec", "patch_generation_ignore_paths": [self.DISTRO_DIR], - "patch_generation_patch_id_digits": self.dist_git.specfile.patch_id_digits, + "patch_generation_patch_id_digits": self.dist_git_specfile.patch_id_digits, "sync_changelog": True, "synced_files": [ { @@ -567,35 +303,40 @@ class SourceGitGenerator: ) def create_from_upstream(self): + """Create a source-git repo, by transforming downstream patches + in Git commits applied on top of the selected upstream ref. """ - create a source-git repo from upstream - """ - if not self.dist_git.specfile.uses_autosetup: + upstream_ref_sha = self.source_git.git.rev_list("-1", self.upstream_ref) + if upstream_ref_sha != self.source_git.head.commit.hexsha: + raise PackitException( + f"{self.upstream_ref!r} is not pointing to the current HEAD " + f"in {self.source_git.working_dir!r}." + ) + if not self.dist_git_specfile.uses_autosetup: raise PackitException( "Initializing source-git repos for packages " "not using %autosetup is not supported." ) - self._pull_upstream_ref() self._populate_distro_dir() self._reset_gitignore() self._configure_syncing() spec = Specfile( f"{self.distro_dir}/{self.pkg_name}.spec", - sources_dir=self.dist_git.local_project.working_dir, + sources_dir=self.dist_git.working_dir, ) patch_comments = spec.read_patch_comments() spec.remove_patches() - self.local_project.stage(path=self.DISTRO_DIR, force=True) - self.local_project.commit( - message="Initialize as a source-git repository", allow_empty=False - ) + self.source_git.git.stage(self.DISTRO_DIR, force=True) + self.source_git.git.commit(message="Initialize as a source-git repository") - self.dist_git.download_source_files(pkg_tool=self.pkg_tool) - self._run_prep() - self._rebase_patches( - get_default_branch(LocalProject(working_dir=self.get_BUILD_dir()).git_repo), - patch_comments, + pkg_tool = PkgTool( + fas_username=self.config.fas_user, + directory=self.dist_git.working_dir, + tool=self.pkg_tool or self.config.pkg_tool, ) + pkg_tool.sources() + self._run_prep() + self._rebase_patches(patch_comments) diff --git a/packit/utils/__init__.py b/packit/utils/__init__.py index b3227b4..0a89797 100644 --- a/packit/utils/__init__.py +++ b/packit/utils/__init__.py @@ -10,8 +10,6 @@ from packit.utils.logging import ( commits_to_nice_str, ) from packit.utils.repo import ( - clone_centos_8_package, - clone_centos_9_package, get_default_branch, get_namespace_and_repo_name, get_repo, @@ -36,8 +34,6 @@ __all__ = [ get_namespace_and_repo_name.__name__, is_a_git_ref.__name__, git_remote_url_to_https_url.__name__, - clone_centos_8_package.__name__, - clone_centos_9_package.__name__, get_default_branch.__name__, ] diff --git a/packit/utils/repo.py b/packit/utils/repo.py index e3c54f5..2be9eff 100644 --- a/packit/utils/repo.py +++ b/packit/utils/repo.py @@ -12,7 +12,6 @@ import yaml import subprocess from ogr.parsing import RepoUrl, parse_git_repo -from packit.constants import CENTOS_DOMAIN, CENTOS_STREAM_GITLAB from packit.utils.commands import run_command from packit.exceptions import PackitException @@ -201,52 +200,6 @@ def get_current_version_command( ] -def clone_centos_8_package( - package_name: str, - dist_git_path: Path, - branch: str = "c8s", - namespace: str = "rpms", - stg: bool = False, -): - """ - clone selected package from git.[stg.]centos.org - """ - run_command( - [ - "git", - "clone", - "-b", - branch, - f"https://git{'.stg' if stg else ''}.{CENTOS_DOMAIN}/{namespace}/{package_name}.git", - str(dist_git_path), - ] - ) - - -def clone_centos_9_package( - package_name: str, - dist_git_path: Path, - branch: str = "c9s", - namespace: str = "rpms", - stg: bool = None, -): - """ - clone selected package from git.[stg.]centos.org - """ - if stg: - logger.warning("There is no staging instance for CentOS Stream 9 dist-git.") - run_command( - [ - "git", - "clone", - "-b", - branch, - f"https://{CENTOS_STREAM_GITLAB}/{namespace}/{package_name}.git", - str(dist_git_path), - ] - ) - - def clone_fedora_package( package_name: str, dist_git_path: Path, diff --git a/tests/data/rpms/hello/hello.spec b/tests/data/rpms/hello/hello.spec new file mode 100644 index 0000000..cbe7669 --- /dev/null +++ b/tests/data/rpms/hello/hello.spec @@ -0,0 +1,28 @@ +Name: hello +Version: 1.0.1 +Release: 1%{?dist} +Summary: Hello World application written in Rust +License: MIT + +URL: src/hello +Source: hello-1.0.1.tar.gz + +# Say hello to Fedora +Patch: turn-into-fedora.patch + +Provides: hello + +BuildRequires: make +BuildRequires: rust + +%description +Hello World application written in Rust + +%prep +%autosetup + +%build +make hello + +%install +install -Dpm0777 hello %{_bindir}/hello diff --git a/tests/data/rpms/hello/sources b/tests/data/rpms/hello/sources new file mode 100644 index 0000000..ee3a80d --- /dev/null +++ b/tests/data/rpms/hello/sources @@ -0,0 +1 @@ +SHA512 (hello-1.0.1.tar.gz) = f101e27058c959f4c412f475c3fc77a90d1ead8728701e4ce04ff08b34139d35e0e72278c9ac7622ba6054e81c0aeca066e09491b5f5666462e3866705a0e892 diff --git a/tests/data/rpms/hello/turn-into-fedora.patch b/tests/data/rpms/hello/turn-into-fedora.patch new file mode 100644 index 0000000..db053cc --- /dev/null +++ b/tests/data/rpms/hello/turn-into-fedora.patch @@ -0,0 +1,9 @@ +diff --git a/hello.rs b/hello.rs +index 47ad8c6..f8dbb5e 100644 +--- a/hello.rs ++++ b/hello.rs +@@ -1,3 +1,3 @@ + fn main() { +- println!("Hello World!"); ++ println!("Hello Fedora!"); + } diff --git a/tests/data/src/.gitignore b/tests/data/src/.gitignore new file mode 100644 index 0000000..ce01362 --- /dev/null +++ b/tests/data/src/.gitignore @@ -0,0 +1 @@ +hello diff --git a/tests/data/src/hello/Makefile b/tests/data/src/hello/Makefile new file mode 100644 index 0000000..fbadaaf --- /dev/null +++ b/tests/data/src/hello/Makefile @@ -0,0 +1,7 @@ +hello: hello.rs + rustc hello.rs + +clean: + rm hello + +.PHONY: clean diff --git a/tests/data/src/hello/README.md b/tests/data/src/hello/README.md new file mode 100644 index 0000000..5978452 --- /dev/null +++ b/tests/data/src/hello/README.md @@ -0,0 +1,3 @@ +# Hello + +Example upstream project written in Rust. diff --git a/tests/data/src/hello/hello.rs b/tests/data/src/hello/hello.rs new file mode 100644 index 0000000..47ad8c6 --- /dev/null +++ b/tests/data/src/hello/hello.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello World!"); +} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6987bb7..bf96ccf 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,24 +1,5 @@ -# MIT License -# -# Copyright (c) 2019 Red Hat, Inc. - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT import datetime import io @@ -28,6 +9,7 @@ import tarfile from pathlib import Path from typing import Iterator, Optional +import git import pytest from flexmock import flexmock from gnupg import GPG @@ -57,11 +39,13 @@ from tests.spellbook import ( initiate_git_repo, DISTGIT, NAME_VERSION, + DATA_DIR, ) DOWNSTREAM_PROJECT_URL = "https://src.fedoraproject.org/not/set.git" UPSTREAM_PROJECT_URL = "https://github.com/also-not/set.git" SOURCE_GIT_RELEASE_TAG = "0.1.0" +HELLO_RELEASE = "1.0.1" @pytest.fixture() @@ -211,6 +195,42 @@ def sourcegit_and_remote(tmp_path): @pytest.fixture() +def source_git_repo(sourcegit_and_remote): + source_git_dir, _ = sourcegit_and_remote + source_git_repo = git.Repo(source_git_dir) + return source_git_repo + + +@pytest.fixture() +def dist_git_repo(distgit_and_remote): + dist_git_dir, _ = distgit_and_remote + dist_git_repo = git.Repo(dist_git_dir) + return dist_git_repo + + +@pytest.fixture() +def hello_source_git_repo(tmp_path): + repo_dir = tmp_path / "src" / "hello" + shutil.copytree(DATA_DIR / "src" / "hello", repo_dir) + repo = git.Repo.init(repo_dir) + repo.git.add(".") + repo.git.commit(message="Initial commit") + repo.git.tag(HELLO_RELEASE) + repo.create_remote("origin", "https://example.com/hello.git") + return repo + + +@pytest.fixture() +def hello_dist_git_repo(tmp_path): + repo_dir = tmp_path / "rpms" / "hello" + shutil.copytree(DATA_DIR / "rpms" / "hello", repo_dir) + repo = git.Repo.init(repo_dir) + repo.git.add(".") + repo.git.commit(message="Initial commit") + return repo + + +@pytest.fixture() def downstream_n_distgit(tmp_path): d_remote = tmp_path / "downstream_remote" d_remote.mkdir() diff --git a/tests/integration/test_source_git_generator.py b/tests/integration/test_source_git_generator.py deleted file mode 100644 index 40255e1..0000000 --- a/tests/integration/test_source_git_generator.py +++ /dev/null @@ -1,488 +0,0 @@ -# Copyright Contributors to the Packit project. -# SPDX-License-Identifier: MIT - -import fileinput -import re -import subprocess -from pathlib import Path -import yaml - -import pytest -import git - -from packit.constants import CENTOS_STREAM_GITLAB -from packit.pkgtool import PkgTool -from packit.local_project import LocalProject -from packit.patches import PatchMetadata -from packit.source_git import SourceGitGenerator -from packit.specfile import Specfile -from packit.utils.repo import create_new_repo, clone_centos_9_package -from packit.exceptions import PackitException -from tests.spellbook import initiate_git_repo - -UNIVERSAL_PACKAGE_NAME = "redhat-rpm-config" - - -@pytest.mark.parametrize( - "fedora_package, centos_package, branch", - ( - (UNIVERSAL_PACKAGE_NAME, None, "main"), - (None, UNIVERSAL_PACKAGE_NAME, "c8s"), - ), -) -def test_distgit_cloning( - api_instance_source_git, fedora_package, centos_package, branch, tmp_path: Path -): - sgg = SourceGitGenerator( - api_instance_source_git.upstream_local_project, - api_instance_source_git.config, - # yes, this is the upstream repo - f"https://src.fedoraproject.org/rpms/{UNIVERSAL_PACKAGE_NAME}", - fedora_package=fedora_package, - centos_package=centos_package, - dist_git_branch=branch, - tmpdir=tmp_path, - ) - sgg._get_dist_git() - assert tmp_path.joinpath(UNIVERSAL_PACKAGE_NAME, ".git").is_dir() - - -# TODO: test different refs: tag, branch, commit hash -def test_fetch_upstream_ref(api_instance_source_git, tmp_path: Path): - tag = "1.0.0" - s = tmp_path.joinpath("s") - u = tmp_path.joinpath("u") - u.mkdir() - create_new_repo(Path(s), []) - initiate_git_repo(u, tag=tag) - sgg = SourceGitGenerator( - LocalProject(working_dir=s), - api_instance_source_git.config, - str(u), - upstream_ref=tag, - centos_package="x", - ) - - sgg._pull_upstream_ref() - - assert s.joinpath(".git").is_dir() - assert sgg.local_project.ref == "main" - assert sgg.local_project.working_dir.joinpath("hops").read_text() == "Cascade\n" - assert sgg.local_project.git_repo.head.commit.message == "commit with data\n" - - -@pytest.mark.parametrize( - "fedora_package, centos_package, branch", - (("fuse-overlayfs", None, "main"),), -) -def test_run_prep( - api_instance_source_git, fedora_package, centos_package, branch, tmp_path: Path -): - sgg = SourceGitGenerator( - api_instance_source_git.upstream_local_project, - api_instance_source_git.config, - f"https://github.com/containers/{fedora_package}", - fedora_package=fedora_package, - centos_package=centos_package, - dist_git_branch=branch, - tmpdir=tmp_path, - ) - sgg.dist_git.download_source_files() - sgg._run_prep() - build_dir = sgg.dist_git.local_project.working_dir.joinpath("BUILD") - assert build_dir.exists() - project_dir = next(build_dir.glob("fuse-overlayfs-*")) - assert project_dir.joinpath(".git") - - -def test_source_git_config_upstream_project_url( - api_instance_source_git, tmp_path: Path -): - """ - use requre to create a source-git out of it in an empty git repo - packit - will pull upstream git history - """ - # requre upstream_project_url - upstream_project_url = "https://github.com/packit/requre.git" - - # clone dist-git - pkg = "python-requre" - dist_git_ref = "6b27ffacda06289ca2d546e15b3c96845243005f" - dist_git_path = tmp_path.joinpath(pkg) - source_git_path = tmp_path.joinpath("requre-sg") - PkgTool().clone(pkg, str(dist_git_path), anonymous=True) - - # check out specific ref - subprocess.check_call(["git", "reset", "--hard", dist_git_ref], cwd=dist_git_path) - - # create src-git - source_git_path.mkdir() - create_new_repo(Path(source_git_path), []) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - upstream_project_url, - upstream_ref="0.4.0", - dist_git_path=dist_git_path, - pkg_name="python-requre", - ) - sgg.create_from_upstream() - - config_file = Path(source_git_path / ".distro/source-git.yaml").read_text() - # black sucks here :/ we are making sure here the yaml looks nice - assert "patch_generation_ignore_paths:\n- .distro\n" in config_file - assert "sources:\n- path: requre-0.4.0.tar.gz\n" in config_file - packit_yaml = yaml.safe_load(config_file) - assert packit_yaml.get("upstream_project_url") == upstream_project_url - - -def test_source_git_config_sources(api_instance_source_git, tmp_path: Path): - """ - use requre to create a source-git out of it in an empty git repo - packit - will pull upstream git history - """ - # requre lookaside_cache_url - requre_tar_url = ( - "https://src.fedoraproject.org/repo/pkgs/rpms/python-requre/" - "requre-0.4.0.tar.gz/sha512/85293577f56e19dd0fad13bb5e118ac2ab39d7" - "570d640a754b9d8d8054078c89c949d4695ef018915b17a2f2428f1635032352d" - "cf3c9a036a2d633013cc35dd9/requre-0.4.0.tar.gz" - ) - requre_tar_path = "requre-0.4.0.tar.gz" - - # clone dist-git - pkg = "python-requre" - dist_git_ref = "6b27ffacda06289ca2d546e15b3c96845243005f" - dist_git_path = tmp_path.joinpath(pkg) - source_git_path = tmp_path.joinpath("requre-sg") - PkgTool().clone(pkg, str(dist_git_path), anonymous=True) - - # check out specific ref - subprocess.check_call(["git", "reset", "--hard", dist_git_ref], cwd=dist_git_path) - - # create src-git - source_git_path.mkdir() - create_new_repo(Path(source_git_path), []) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - "https://github.com/packit/requre", - upstream_ref="0.4.0", - dist_git_path=dist_git_path, - pkg_name="python-requre", - ) - sgg.create_from_upstream() - - config_file = Path(source_git_path / ".distro/source-git.yaml").read_text() - # black sucks here :/ we are making sure here the yaml looks nice - assert "patch_generation_ignore_paths:\n- .distro\n" in config_file - assert "sources:\n- path: requre-0.4.0.tar.gz\n" in config_file - packit_yaml = yaml.safe_load(config_file) - assert len(packit_yaml.get("sources", {})) > 0 - assert packit_yaml["sources"][0].get("url") == requre_tar_url - assert packit_yaml["sources"][0].get("path") == requre_tar_path - - -REQURE_PATCH = r"""\ -diff --git a/README.md b/README.md -index 17e9e85..99ef68c 100644 ---- a/README.md -+++ b/README.md -@@ -15,3 +15,5 @@ back - - Used for testing [packit-service](https://github.com/packit-service) organization projects - - ogr - - packit -+ -+Hello! -""" - - -def test_create_srcgit_requre_clean(api_instance_source_git, tmp_path: Path): - """ - use requre to create a source-git out of it in an empty git repo - packit - will pull upstream git history - """ - # clone dist-git - pkg = "python-requre" - dist_git_ref = "6b27ffacda06289ca2d546e15b3c96845243005f" - dist_git_path = tmp_path.joinpath(pkg) - source_git_path = tmp_path.joinpath("requre-sg") - PkgTool().clone(pkg, str(dist_git_path), anonymous=True) - dg_lp = LocalProject(working_dir=dist_git_path) - - # check out specific ref - subprocess.check_call(["git", "reset", "--hard", dist_git_ref], cwd=dist_git_path) - - # add a patch in there - spec = Specfile(dist_git_path / f"{pkg}.spec", sources_dir=dist_git_path) - patch_name = "hello.patch" - patch_path = dist_git_path.joinpath(patch_name) - patch_path.write_text(REQURE_PATCH) - patch = PatchMetadata(name=patch_name, path=patch_path, present_in_specfile=False) - spec.add_patch(patch) - dg_lp.stage() - dg_lp.commit("add the hello patch") - subprocess.check_call(["fedpkg", "prep"], cwd=dist_git_path) - # Clean the dist-git repo before initializing a source-git repo from it - # so that only checked-in content ends up in source-git. - dg_lp.git_repo.git.clean("-xdff") - - # create src-git - source_git_path.mkdir() - create_new_repo(Path(source_git_path), []) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - "https://github.com/packit/requre", - upstream_ref="0.4.0", - dist_git_path=dist_git_path, - pkg_name="python-requre", - ) - sgg.create_from_upstream() - - # Verify it! - # Updating the dist-git repo from this source-git one, - # should result in no changes. - # First make sure the dist-git repo is clean. - dg_lp.git_repo.git.clean("-xdff") - # Now convert. - subprocess.check_call( - [ - "packit", - "--config", - ".distro/source-git.yaml", - "source-git", - "update-dist-git", - ".", - str(dist_git_path), - ], - cwd=source_git_path, - ) - # There are no changes or untracked files. - dg_lp.git_repo.git.diff(exit_code=True) - assert not dg_lp.git_repo.git.clean("-xdn") - - # verify the archive is not committed in the source-git - with pytest.raises(subprocess.CalledProcessError) as exc: - subprocess.check_call( - [ - "git", - "ls-files", - "--error-unmatch", - f".distro/{spec.get_archive()}", - ], - cwd=source_git_path, - ) - assert exc.value.returncode == 1 - - -def test_create_srcgit_requre_populated(api_instance_source_git, tmp_path: Path): - """ - use requre to create a source-git out of it in a branch with upstream git history - - this should only layer downstream changes on top - """ - # clone dist-git - pkg = "python-requre" - dist_git_ref = "6b27ffacda06289ca2d546e15b3c96845243005f" - dist_git_path = tmp_path.joinpath(pkg) - source_git_path = tmp_path.joinpath("requre-sg") - PkgTool().clone(pkg, str(dist_git_path), anonymous=True) - dg_lp = LocalProject(working_dir=dist_git_path) - - # check out specific ref - subprocess.check_call(["git", "reset", "--hard", dist_git_ref], cwd=dist_git_path) - - # add a patch in there - spec = Specfile(dist_git_path / f"{pkg}.spec", sources_dir=dist_git_path) - patch_name = "hello.patch" - patch_path = dist_git_path.joinpath(patch_name) - patch_path.write_text(REQURE_PATCH) - patch = PatchMetadata(name=patch_name, path=patch_path, present_in_specfile=False) - spec.add_patch(patch) - dg_lp.stage() - dg_lp.commit("add the hello patch") - subprocess.check_call(["fedpkg", "prep"], cwd=dist_git_path) - # Clean the dist-git repo before initializing a source-git repo from it - # so that only checked-in content ends up in source-git. - dg_lp.git_repo.git.clean("-xdff") - - # create src-git - source_git_path.mkdir() - subprocess.check_call( - ["git", "clone", "https://github.com/packit/requre", str(source_git_path)] - ) - subprocess.check_call( - ["git", "checkout", "-B", "source-git-0.4.0", "0.4.0"], cwd=source_git_path - ) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - upstream_ref="0.4.0", - dist_git_path=dist_git_path, - pkg_name="python-requre", - ) - sgg.create_from_upstream() - - # Verify it! - # Updating the dist-git repo from this source-git one, - # should result in no changes. - # First make sure the dist-git repo is clean. - dg_lp.git_repo.git.clean("-xdff") - # Now convert. - subprocess.check_call( - [ - "packit", - "--config", - ".distro/source-git.yaml", - "source-git", - "update-dist-git", - ".", - str(dist_git_path), - ], - cwd=source_git_path, - ) - # There are no changes or untracked files. - dg_lp.git_repo.git.diff(exit_code=True) - assert not dg_lp.git_repo.git.clean("-xdn") - - -@pytest.mark.slow -@pytest.mark.parametrize("dist_git_branch,upstream_ref", [("c9s", "cronie-1.5.5")]) -def test_centos_cronie( - dist_git_branch, upstream_ref, api_instance_source_git, tmp_path: Path -): - source_git_path = tmp_path.joinpath("cronie-sg") - # create src-git - source_git_path.mkdir() - create_new_repo(source_git_path, []) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - upstream_url="https://github.com/cronie-crond/cronie", - upstream_ref=upstream_ref, - centos_package="cronie", - dist_git_branch=dist_git_branch, - pkg_name="cronie", - ) - sgg.create_from_upstream() - - assert CENTOS_STREAM_GITLAB in sgg.dist_git.local_project.git_url - - # verify it - # TODO: this should be updated to be verified with update-dist-git also. Unfortunately - # this currently produces some change in dist-git b/c of how the Patch is written in the - # spec-file. - subprocess.check_call( - ["packit", "--config", ".distro/source-git.yaml", "srpm"], cwd=source_git_path - ) - srpm_path = list(source_git_path.glob("cronie-*.src.rpm"))[0] - assert srpm_path.is_file() - - -@pytest.mark.slow -@pytest.mark.parametrize("apply_option", ("git", "git_am")) -def test_acl_with_git_git_am(apply_option, api_instance_source_git, tmp_path: Path): - """manipulate acl's dist-git to use -Sgit and -Sgit_am so we can verify it works""" - dist_git_branch = "c9s" - package_name = "acl" - source_git_path = tmp_path.joinpath("acl-sg") - # dist-git tools expect: dir name == package name - dist_git_path = tmp_path.joinpath(package_name) - - # create src-git - source_git_path.mkdir() - create_new_repo(source_git_path, []) - - # fetch dist-git and change %autosetup - clone_centos_9_package( - package_name, - dist_git_path=dist_git_path, - branch=dist_git_branch, - ) - for line in fileinput.input( - dist_git_path.joinpath(f"{package_name}.spec"), inplace=True - ): - if "%autosetup" in line: - line = f"%autosetup -p1 -S{apply_option}" - print(line, end="") # \n would make double-newlines here - - # run conversion - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - "https://git.savannah.nongnu.org/git/acl.git", - upstream_ref="v2.3.1", - centos_package=package_name, - dist_git_branch=dist_git_branch, - dist_git_path=dist_git_path, - pkg_name=package_name, - ) - sgg.create_from_upstream() - - # verify the patch commit has metadata - patch_commit_message = sgg.local_project.git_repo.head.commit.message - assert "present_in_specfile: true" in patch_commit_message - assert "patch_name: 0001-acl-2.2.53-test-runwrapper.patch" in patch_commit_message - assert re.findall(r"patch_id: \d", patch_commit_message) - - -def test_check_for_autosetup(api_instance_source_git, tmp_path): - """Check if the package is using %autosetup before doing the conversion""" - package_name = "ed" - source_git_path = tmp_path / "src" / package_name - dist_git_path = tmp_path / "rpms" / package_name - - source_git_path.mkdir(parents=True) - create_new_repo(source_git_path, []) - - dist_git_path.mkdir(parents=True) - create_new_repo(dist_git_path, []) - dist_git_repo = git.Repo(dist_git_path) - dist_git_repo.git.remote( - "add", "origin", "https://gitlab.com/redhat/centos-stream/rpms/ed.git" - ) - - (dist_git_path / f"{package_name}.spec").write_text( - """\ -Summary: The GNU line editor -Name: ed -Version: 1.14.2 -Release: 11%{?dist} -License: GPLv3+ and GFDL -Source: %{name}-%{version}.tar.xz - -%description -Ed is a line-oriented text editor, used to create, display, and modify - -%prep -%setup -q - -%build -%configure -%make_build CFLAGS="%{optflags}" LDFLAGS="%{__global_ldflags}" - -%install -%make_install -rm -vrf %{buildroot}%{_infodir}/dir - -%files -%license COPYING -%doc ChangeLog NEWS README TODO AUTHORS -%{_bindir}/ed -%{_bindir}/red -%{_mandir}/man1/ed.1* -%{_mandir}/man1/red.1* -%{_infodir}/ed.info* - -%changelog -""" - ) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - api_instance_source_git.config, - centos_package=package_name, - dist_git_path=dist_git_path, - pkg_name=package_name, - ) - with pytest.raises(PackitException): - sgg.create_from_upstream() diff --git a/tests/integration/test_source_git_init.py b/tests/integration/test_source_git_init.py new file mode 100644 index 0000000..fe4cff2 --- /dev/null +++ b/tests/integration/test_source_git_init.py @@ -0,0 +1,201 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import flexmock +import pytest +import yaml + +from pathlib import Path + +from packit.exceptions import PackitException +from packit.source_git import SourceGitGenerator +from packit.pkgtool import PkgTool + +from tests.integration.conftest import HELLO_RELEASE + + +def test_upstream_ref_not_at_head(hello_source_git_repo, hello_dist_git_repo): + """Initializing a source-git repo fails, if the upstream_ref which + should be the starting point of the downstream changes is not at + the HEAD of the source-git repo. + """ + readme = Path(hello_source_git_repo.working_dir, "README.md") + with open(readme, "a") as fp: + fp.write("\nSome new thing\n") + hello_source_git_repo.git.add("README.md") + hello_source_git_repo.git.commit(message="Add a new thing") + + sgg = SourceGitGenerator( + config=flexmock(), + source_git=hello_source_git_repo, + dist_git=hello_dist_git_repo, + upstream_ref=HELLO_RELEASE, + ) + with pytest.raises(PackitException) as ex: + sgg.create_from_upstream() + assert "not pointing to the current HEAD" in str(ex.value) + + +def test_not_using_autosetup(hello_source_git_repo, hello_dist_git_repo): + """Initializing a source-git repo for packages which don't use + %autosetup in their specfile is currently not supported. + """ + spec = Path(hello_dist_git_repo.working_dir, "hello.spec") + content = spec.read_text().replace("%autosetup", "%setup") + spec.write_text(content) + hello_dist_git_repo.git.add("hello.spec") + hello_dist_git_repo.git.commit(message="Use %setup") + + sgg = SourceGitGenerator( + config=flexmock(), + source_git=hello_source_git_repo, + dist_git=hello_dist_git_repo, + upstream_ref=HELLO_RELEASE, + ) + with pytest.raises(PackitException) as ex: + sgg.create_from_upstream() + assert "not using %autosetup" in str(ex.value) + + +def modify_file(path: Path): + content = path.read_text() + "\n# some change\n" + path.write_text(content) + + +def create_file(path: Path): + Path(path.parent, "new.file").touch() + + +@pytest.mark.parametrize("change", [modify_file, create_file]) +def test_dist_git_not_pristine(hello_source_git_repo, hello_dist_git_repo, change): + """Initialization fails if the dist-git working directory is not + pristine, has changes and/or untracked files, in order to avoid + tracking files in source-git which are not part of the dist-git repo + and history. + """ + change(Path(hello_dist_git_repo.working_dir, "hello.spec")) + + sgg = SourceGitGenerator( + config=flexmock(), + source_git=hello_source_git_repo, + dist_git=hello_dist_git_repo, + upstream_ref=HELLO_RELEASE, + pkg_name="hello", + ) + with pytest.raises(PackitException) as ex: + sgg.create_from_upstream() + assert "is not pristine" in str(ex.value) + + +def download_sources(from_repo, to_repo): + def funk(): + from_repo.git.archive( + HELLO_RELEASE, + format="tar.gz", + prefix=f"hello-{HELLO_RELEASE}/", + output=f"{to_repo.working_dir}/hello-{HELLO_RELEASE}.tar.gz", + ) + + return funk + + +def check_source_git_config(source_git_config): + assert source_git_config["upstream_project_url"] == "https://example.com/hello.git" + assert source_git_config["upstream_ref"] == HELLO_RELEASE + assert source_git_config["downstream_package_name"] == "hello" + assert source_git_config["specfile_path"] == ".distro/hello.spec" + assert source_git_config["patch_generation_ignore_paths"] == [".distro"] + assert source_git_config["sync_changelog"] is True + assert source_git_config["synced_files"] == [ + { + "src": ".distro/", + "dest": ".", + "delete": True, + "filters": [ + "protect .git*", + "protect sources", + "exclude source-git.yaml", + "exclude .gitignore", + ], + } + ] + assert source_git_config["sources"][0]["path"] == f"hello-{HELLO_RELEASE}.tar.gz" + + +def test_create_from_upstream_no_patch(hello_source_git_repo, hello_dist_git_repo): + """A source-git repo is properly initialized from a dist-git repo. + - No downstream patches. + """ + spec = Path(hello_dist_git_repo.working_dir, "hello.spec") + content = [ + line for line in spec.read_text().splitlines() if not line.startswith("Patch") + ] + spec.write_text("\n".join(content)) + Path(hello_dist_git_repo.working_dir, "turn-into-fedora.patch").unlink() + hello_dist_git_repo.git.add(".") + hello_dist_git_repo.git.commit(message="Remove the patch") + + flexmock( + PkgTool, sources=download_sources(hello_source_git_repo, hello_dist_git_repo) + ) + sgg = SourceGitGenerator( + config=flexmock(fas_user=None, pkg_tool="fedpkg"), + source_git=hello_source_git_repo, + dist_git=hello_dist_git_repo, + upstream_ref=HELLO_RELEASE, + pkg_name="hello", + ) + sgg.create_from_upstream() + source_git_config = yaml.safe_load( + Path( + hello_source_git_repo.working_dir, ".distro", "source-git.yaml" + ).read_text() + ) + check_source_git_config(source_git_config) + assert source_git_config["patch_generation_patch_id_digits"] == 1 + + +def test_create_from_upstream_with_patch(hello_source_git_repo, hello_dist_git_repo): + """A source-git repo is properly initialized from a dist-git repo. + - A few downstream patches. + """ + flexmock( + PkgTool, sources=download_sources(hello_source_git_repo, hello_dist_git_repo) + ) + sgg = SourceGitGenerator( + config=flexmock(fas_user=None, pkg_tool="fedpkg"), + source_git=hello_source_git_repo, + dist_git=hello_dist_git_repo, + upstream_ref=HELLO_RELEASE, + pkg_name="hello", + ) + sgg.create_from_upstream() + source_git_config = yaml.safe_load( + Path( + hello_source_git_repo.working_dir, ".distro", "source-git.yaml" + ).read_text() + ) + check_source_git_config(source_git_config) + assert source_git_config["patch_generation_patch_id_digits"] == 0 + + assert ( + Path(hello_source_git_repo.working_dir, ".distro", ".gitignore").read_text() + == """\ +# Reset gitignore rules +!* +""" + ) + assert not Path(hello_source_git_repo.working_dir, ".distro", "sources").exists() + assert not Path(hello_source_git_repo.working_dir, ".distro", ".git").exists() + + assert ( + "Hello Fedora" + in Path(hello_source_git_repo.working_dir, "hello.rs").read_text() + ) + + assert ( + "patch_name: turn-into-fedora.patch" + in hello_source_git_repo.head.commit.message + ) + assert "patch_id: 0" in hello_source_git_repo.head.commit.message + assert "description:" in hello_source_git_repo.head.commit.message diff --git a/tests/unit/test_source_git_generator.py b/tests/unit/test_source_git_generator.py deleted file mode 100644 index 61e90b5..0000000 --- a/tests/unit/test_source_git_generator.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright Contributors to the Packit project. -# SPDX-License-Identifier: MIT -from pathlib import Path - -from flexmock import flexmock - -from packit.config import Config -from packit.distgit import DistGit -from packit.local_project import LocalProject -from packit.source_git import SourceGitGenerator, CentOS8DistGit -from packit.utils.repo import create_new_repo - - -def test_centos_conf(cronie, tmp_path: Path): - """make sure the centos-specific configuration is correct""" - source_git_path = tmp_path.joinpath("cronie-sg") - # create src-git - source_git_path.mkdir() - create_new_repo(source_git_path, []) - sgg = SourceGitGenerator( - LocalProject(working_dir=source_git_path), - Config(), - dist_git_path=cronie, - upstream_ref="cronie-1.5.2", - centos_package="cronie", - ) - - dg = sgg.dist_git - assert isinstance(dg, CentOS8DistGit) - - flexmock( - DistGit, - download_upstream_archive=lambda: cronie / "SOURCES" / "cronie-1.5.2.tar.gz", - ) - assert sgg.primary_archive == cronie / "SOURCES" / "cronie-1.5.2.tar.gz" - - assert dg.absolute_source_dir == cronie / "SOURCES" - assert dg.absolute_specfile_dir == cronie / "SPECS" - assert dg.absolute_specfile_path == cronie / "SPECS" / "cronie.spec"