From 5faef228f299ddd84c0e950ba2364a030a50b513 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Sep 05 2018 16:38:09 +0000 Subject: [PATCH 1/3] Don't pass the MBS API URL around as a parameter Rather than making it a parameter, make it a property of the Commands object that the caller needs to set first. This gives more flexibility when using the API URL - in particular it can be use when loading a property value. We still set it only for MBS commands to avoid validating the MBS API configuration and checking the API version when MBS is not being used. Signed-off-by: Owen W. Taylor --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 696c9c3..e43ff3f 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -191,6 +191,8 @@ class Commands(object): self.lookaside_namespaced = lookaside_namespaced # Deprecates self.module_name self._repo_name = None + # API URL for the module build server + self.module_api_url = None # Define properties here # Properties allow us to "lazy load" various attributes, which also means @@ -2933,13 +2935,12 @@ class Commands(object): cmd.extend([project, srpm_name]) self._run_command(cmd) - def module_build_cancel(self, api_url, build_id, auth_method, + def module_build_cancel(self, build_id, auth_method, oidc_id_provider=None, oidc_client_id=None, oidc_client_secret=None, oidc_scopes=None): """ Cancel an MBS build - :param str api_url: URL of the MBS API :param int build_id: build ID to cancel :param str auth_method: authentication method used by the MBS :kwarg str oidc_id_provider: the OIDC provider when MBS is using OIDC @@ -2953,8 +2954,8 @@ class Commands(object): for authentication """ # Make sure the build they are trying to cancel exists - self.module_get_build(api_url, build_id) - url = self.module_get_url(api_url, build_id, action='PATCH') + self.module_get_build(build_id) + url = self.module_get_url(build_id, action='PATCH') resp = self.module_send_authorized_request( 'PATCH', url, {'state': 'failed'}, auth_method, oidc_id_provider, oidc_client_id, oidc_client_secret, oidc_scopes, timeout=60) @@ -2967,18 +2968,17 @@ class Commands(object): 'The cancellation of module build #{0} failed with:\n{1}' .format(build_id, error_msg)) - def module_build_info(self, api_url, build_id): + def module_build_info(self, build_id): """ Show information about an MBS build - :param str api_url: URL of the MBS API :param int build_id: build ID to query MBS about """ # Load the Koji session anonymously so we get access to the Koji web # URL self.load_kojisession(anon=True) state_names = self.module_get_koji_state_dict() - data = self.module_get_build(api_url, build_id) + data = self.module_get_build(build_id) print('Name: {0}'.format(data['name'])) print('Stream: {0}'.format(data['stream'])) print('Version: {0}'.format(data['version'])) @@ -3000,14 +3000,13 @@ class Commands(object): state_names[task_data.get('state', None)])) print(' Koji Task: {0}\n'.format(koji_task_url)) - def module_get_build(self, api_url, build_id): + def module_get_build(self, build_id): """ Get an MBS build - :param api_url: a string of the URL of the MBS API :param build_id: an integer of the build ID to query MBS about :return: None or a dictionary representing the module build """ - url = self.module_get_url(api_url, build_id) + url = self.module_get_url(build_id) response = requests.get(url, timeout=60) if response.ok: return response.json() @@ -3020,11 +3019,10 @@ class Commands(object): 'The following error occurred while getting information on ' 'module build #{0}:\n{1}'.format(build_id, error_msg)) - def module_get_url(self, api_url, build_id, action='GET'): + def module_get_url(self, build_id, action='GET'): """ Get the proper MBS API URL for the desired action - :param str api_url: a string of the URL of the MBS API :param int build_id: an integer of the module build desired. If this is set to None, then the base URL for all module builds is returned. :param str action: a string determining the HTTP action. If this is set @@ -3033,7 +3031,7 @@ class Commands(object): :return: a string of the desired MBS API URL. :rtype: str """ - url = urljoin(api_url, 'module-builds/') + url = urljoin(self.module_api_url, 'module-builds/') if build_id is not None: url = '{0}{1}'.format(url, build_id) else: @@ -3172,11 +3170,10 @@ class Commands(object): else: raise - def module_overview(self, api_url, limit=10, finished=True): + def module_overview(self, limit=10, finished=True): """ Show the overview of the latest builds in MBS - :param str api_url: a string of the URL of the MBS API :param int limit: an integer of the number of most recent module builds to display. This defaults to 10. :param bool finished: a boolean that determines if only finished or @@ -3193,7 +3190,7 @@ class Commands(object): 'failed': 4, 'ready': 5, } - baseurl = self.module_get_url(api_url, build_id=None) + baseurl = self.module_get_url(build_id=None) if finished: # These are the states when a build is finished states = [build_states['done'], build_states['ready'], @@ -3339,14 +3336,13 @@ class Commands(object): raise rpkgError('An unsupported MBS "auth_method" was provided') return resp - def module_submit_build(self, api_url, scm_url, branch, auth_method, + def module_submit_build(self, scm_url, branch, auth_method, optional=None, oidc_id_provider=None, oidc_client_id=None, oidc_client_secret=None, oidc_scopes=None): """ Submit a module build to the MBS - :param api_url: a string of the URL of the MBS API :param scm_url: a string of the module's SCM URL :param branch: a string of the module's branch :param str auth_method: a string of the authentication method used by @@ -3378,7 +3374,7 @@ class Commands(object): 'Optional arguments are not in the proper "key=value" format') body.update(optional_dict) - url = self.module_get_url(api_url, build_id=None, action='POST') + url = self.module_get_url(build_id=None, action='POST') resp = self.module_send_authorized_request( 'POST', url, body, auth_method, oidc_id_provider, oidc_client_id, oidc_client_secret, oidc_scopes, timeout=120) @@ -3399,13 +3395,12 @@ class Commands(object): builds = data if isinstance(data, list) else [data] return [build['id'] for build in builds] - def module_watch_build(self, api_url, build_ids): + def module_watch_build(self, build_ids): """ Watches the first MBS build in the list in a loop that updates every 15 seconds. The loop ends when the build state is 'failed', 'done', or 'ready'. - :param str api_url: a string of the URL of the MBS API :param build_ids: a list of module build IDs :type build_ids: list[int] """ @@ -3423,7 +3418,7 @@ class Commands(object): done = False while not done: state_names = self.module_get_koji_state_dict() - build = self.module_get_build(api_url, build_id) + build = self.module_get_build(build_id) tasks = {} if 'rpms' in build['tasks']: tasks = build['tasks']['rpms'] @@ -3494,6 +3489,10 @@ class Commands(object): :param api_url: a string of the URL of the MBS API :return: an int of the API version """ + + # We don't use self.module_api_url since this is used exclusively by the code + # that is loading and validating the API URL before setting it. + url = '{0}/about/'.format(api_url.rstrip('/')) response = requests.get(url, timeout=60) if response.ok: diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index 44beb9c..f6528d7 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -1923,7 +1923,7 @@ see API KEY section of copr-cli(1) man page. def module_build(self): """Builds a module using MBS""" - api_url = self.module_api_url + self.set_module_api_url() self.module_validate_config() scm_url, branch = self.cmd.module_get_scm_info( self.args.scm_url, self.args.branch) @@ -1933,7 +1933,7 @@ see API KEY section of copr-cli(1) man page. if not self.args.q: print('Submitting the module build...') build_ids = self._cmd.module_submit_build( - api_url, scm_url, branch, auth_method, self.args.optional, + scm_url, branch, auth_method, self.args.optional, oidc_id_provider, oidc_client_id, oidc_client_secret, oidc_scopes) if self.args.watch: self.module_watch_build(build_ids) @@ -1948,7 +1948,7 @@ see API KEY section of copr-cli(1) man page. def module_build_cancel(self): """Cancel an MBS build""" - api_url = self.module_api_url + self.set_module_api_url() build_id = self.args.build_id auth_method, oidc_id_provider, oidc_client_id, oidc_client_secret, \ oidc_scopes = self.module_get_auth_config() @@ -1956,14 +1956,15 @@ see API KEY section of copr-cli(1) man page. if not self.args.q: print('Cancelling module build #{0}...'.format(build_id)) self.cmd.module_build_cancel( - api_url, build_id, auth_method, oidc_id_provider, oidc_client_id, + build_id, auth_method, oidc_id_provider, oidc_client_id, oidc_client_secret, oidc_scopes) if not self.args.q: print('The module build #{0} was cancelled'.format(build_id)) def module_build_info(self): """Show information about an MBS build""" - self.cmd.module_build_info(self.module_api_url, self.args.build_id) + self.set_module_api_url() + self.cmd.module_build_info(self.args.build_id) def module_build_local(self): """Build a module locally using mbs-manager""" @@ -2072,14 +2073,18 @@ see API KEY section of copr-cli(1) man page. api_url.rstrip('/'), api_version) return self._module_api_url + def set_module_api_url(self): + self.cmd.module_api_url = self.module_api_url + def module_build_watch(self): """Watch an MBS build from the command-line""" self.module_watch_build([self.args.build_id]) def module_overview(self): """Show the overview of the latest builds in the MBS""" + self.set_module_api_url() self.cmd.module_overview( - self.module_api_url, self.args.limit, + self.args.limit, finished=(not self.args.unfinished)) def module_validate_config(self): @@ -2134,7 +2139,8 @@ see API KEY section of copr-cli(1) man page. :param build_ids: a list of module build IDs :type build_ids: list[int] """ - self.cmd.module_watch_build(self.module_api_url, build_ids) + self.set_module_api_url() + self.cmd.module_watch_build(build_ids) def new(self): new_diff = self.cmd.new() From 387ecf1eeb9eae584c51ab70ac4ea56d55a29dfb Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Sep 13 2018 14:58:58 +0000 Subject: [PATCH 2/3] Add flatpak-build subcommand The flatpak-build subcommand starts a build of the current git repository as a Flatpak. It's pretty much identical to container-build, but the build target is determined by looking up the module that will be built into the flatpak in container.yaml and then determining what platform version that uses by traversing module dependencies. Signed-off-by: Owen W. Taylor --- diff --git a/etc/bash_completion.d/rpkg.bash b/etc/bash_completion.d/rpkg.bash index 9dd8de3..c564afc 100644 --- a/etc/bash_completion.d/rpkg.bash +++ b/etc/bash_completion.d/rpkg.bash @@ -34,8 +34,8 @@ _rpkg() local options="--help -v -q" local options_value="--dist --release --user --path" - local commands="build chain-build ci clean clog clone co container-build container-build-config commit compile copr-build diff gimmespec giturl help \ - gitbuildhash import install lint local mockbuild mock-config new new-sources patch prep pull push scratch-build sources \ + local commands="build chain-build ci clean clog clone co container-build container-build-config commit compile copr-build diff flatpak-build \ + gimmespec giturl help gitbuildhash import install lint local mockbuild mock-config new new-sources patch prep pull push scratch-build sources \ srpm switch-branch tag unused-patches upload verify-files verrel" # parse main options and get command @@ -152,6 +152,12 @@ _rpkg() after="file" after_more=true ;; + flatpak-build) + options="--scratch --nowait" + options_target="--target" + options_string="--repo-url" + options_arches="--arches" + ;; import) options="--create" options_branch="--branch" diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index e43ff3f..26d637a 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -39,6 +39,7 @@ from six.moves import configparser from six.moves import urllib from six.moves.urllib.parse import urljoin import requests +import yaml from pyrpkg.errors import HashtypeMixingError, rpkgError, rpkgAuthError, \ UnknownTargetError @@ -47,6 +48,10 @@ from pyrpkg.lookaside import CGILookasideCache from pyrpkg.sources import SourcesFile from pyrpkg.utils import cached_property, log_result, find_me +import gi +gi.require_version('Modulemd', '1.0') +from gi.repository import Modulemd # noqa + class NullHandler(logging.Handler): """Null logger to avoid spurious messages, add a handler in app code""" @@ -156,6 +161,8 @@ class Commands(object): self._target = target # The build target for containers within the buildsystem self._container_build_target = target + # The build target for flatpaks within the buildsystem + self._flatpak_build_target = target # The top url to our build server self._topurl = None # The user to use or discover @@ -890,6 +897,98 @@ class Commands(object): self._container_build_target = '%s-%s-candidate' % (self.branch_merge, self.ns) @property + def flatpak_build_target(self): + """This property ensures the target for flatpak builds.""" + if not self._flatpak_build_target: + self.load_flatpak_build_target() + return self._flatpak_build_target + + def _find_platform_stream(self, name, stream, version=None): + """Recursively search for the platform module in dependencies to find its stream. + + The stream of the 'platform' pseudo-module determines what base package set + we need for the runtime - and thus what build target we need. + """ + + if version is not None: + nsvc = name + ':' + stream + ':' + version + else: + nsvc = name + ':' + stream + + build = self.module_get_latest_build(nsvc) + + if build is None: + raise rpkgError("Cannot find any builds for module %s" % nsvc) + + mmd_str = build['modulemd'] + + objects = Modulemd.objects_from_string(mmd_str) + modules = [o for o in objects if isinstance(o, Modulemd.Module)] + if len(modules) != 1: + raise rpkgError("Failed to load modulemd for %s" % nsvc) + + # Streams should already be expanded in the modulemd's that we retrieve + # from MBS - modules were built against a particular dependency. + def get_stream(req, req_stream): + req_stream_list = req_stream.get() + if len(req_stream_list) != 1: + raise rpkgError("%s: stream list for '%s' is not expanded (%s)" % + (nsvc, req, req_stream_list)) + return req_stream_list[0] + + # We first look for 'platform' as a direct dependency of this module, + # before recursing into the dependencies + for dep in modules[0].peek_dependencies(): + for req, req_stream in dep.peek_requires().items(): + if req == 'platform': + return get_stream(req, req_stream) + + # Now recurse into the dependencies + for dep in modules[0].peek_dependencies(): + for req, req_stream in dep.peek_requires().items(): + platform_stream = self._find_platform_stream(req, + get_stream(req, req_stream)) + if platform_stream: + return platform_stream + + return None + + def load_flatpak_build_target(self): + """This locates a target appropriate for the runtime that the Flatpak targets.""" + + # Find the module we are going to build from container.yaml + yaml_path = os.path.join(self.path, "container.yaml") + if not os.path.exists(yaml_path): + raise rpkgError("Cannot find 'container.yaml' to determine build target.") + + with open(yaml_path) as f: + container_yaml = yaml.safe_load(f) + + compose = container_yaml.get('compose', {}) + modules = compose.get('modules', []) + if not modules: + raise rpkgError("No modules listed in 'container.yaml'") + if len(modules) > 1: + raise rpkgError("Multiple modules listed in 'container.yaml'") + module = modules[0] + + parts = module.split(':') + if len(parts) == 2: + name, stream = parts + version = None + elif len(parts) == 3: + name, stream, version = parts + else: + raise rpkgError("Module in container.yaml should be NAME:STREAM[:VERSION]") + + platform_stream = self._find_platform_stream(name, stream, version=version) + if platform_stream is None: + raise rpkgError("Unable to find 'platform' module in the dependencies of '%s'; " + "can't determine target" % module) + + self._flatpak_build_target = '%s-flatpak-candidate' % platform_stream + + @property def topurl(self): """This property ensures the topurl attribute""" @@ -2826,10 +2925,13 @@ class Commands(object): kojiconfig=None, kojiprofile=None, build_client=None, koji_task_watcher=None, - nowait=False): + nowait=False, + flatpak=False): # check if repo is dirty and all commits are pushed self.check_repo() - container_target = self.target if target_override else self.container_build_target + container_target = self.target if target_override \ + else self.flatpak_build_target if flatpak \ + else self.container_build_target # This is for backward-compatibility of deprecated kojiconfig. # Signature of container_build_koji is not changed in case someone @@ -2873,6 +2975,9 @@ class Commands(object): raise rpkgError('Cannot override arches for non-scratch builds') task_opts['arch_override'] = ' '.join(arches) + if flatpak: + task_opts['flatpak'] = True + priority = opts.get("priority", None) task_id = self.kojisession.buildContainer(source, container_target, @@ -3019,6 +3124,38 @@ class Commands(object): 'The following error occurred while getting information on ' 'module build #{0}:\n{1}'.format(build_id, error_msg)) + def module_get_latest_build(self, nsvc): + """ + Get the latest MBS build for a particular module. If the module is + built with multiple contexts, a random one will be returned. + + :param nsvc: a NAME:STREAM:VERSION:CONTEXT to filter the query + (may be partial - e.g. only NAME or only NAME:STREAM) + :return: the latest build + """ + url = self.module_get_url(None) + params = { + 'nsvc': nsvc, + 'order_desc_by': 'version', + 'per_page': 1 + } + + response = requests.get(url, timeout=60, params=params) + if response.ok: + j = response.json() + if len(j['items']) == 0: + return None + else: + return j['items'][0] + else: + try: + error_msg = response.json()['message'] + except (ValueError, KeyError): + error_msg = response.text + raise rpkgError( + 'The following error occurred while getting information on ' + 'module #{0}:\n{1}'.format(nsvc, error_msg)) + def module_get_url(self, build_id, action='GET'): """ Get the proper MBS API URL for the desired action diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index f6528d7..128d516 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -424,6 +424,7 @@ class cliClient(object): # Add a common parsers self.register_build_common() + self.register_container_build_common() self.register_rpm_common() # Other targets @@ -438,6 +439,7 @@ class cliClient(object): self.register_container_build() self.register_container_build_setup() self.register_diff() + self.register_flatpak_build() self.register_gimmespec() self.register_gitbuildhash() self.register_gitcred() @@ -1379,11 +1381,54 @@ see API KEY section of copr-cli(1) man page. 'verrel', help='Print the name-version-release') verrel_parser.set_defaults(command=self.verrel) + def register_container_build_common(self): + parser = ArgumentParser( + 'container_build_common', add_help=False, allow_abbrev=False) + + self.container_build_parser_common = parser + + parser.add_argument( + '--target', + help='Override the default target', + default=None) + + parser.add_argument( + '--nowait', + action='store_true', + default=False, + help="Don't wait on build") + + parser.add_argument( + '--scratch', + help='Scratch build', + action="store_true") + + parser.add_argument( + '--arches', + action='store', + nargs='*', + help='Limit a scratch build to an arch. May have multiple arches.') + + parser.add_argument( + '--skip-remote-rules-validation', + action='store_true', + default=False, + help="Don't check if there's a valid gating.yaml file in the repo") + def register_container_build(self): self.container_build_parser = self.subparsers.add_parser( 'container-build', help='Build a container', - description='Build a container') + description='Build a container', + parents=[self.container_build_parser_common]) + + # These arguments are specific to non-Flatpak containers + # + # --compose-id is implemented for Flatpaks as a side-effect of the internal + # implementation, but it is unlikely to be useful to trigger through rpkg. + # --signing-intent is not implemented for Flatpaks, though it could be useful + # --repo-url makes no sense for flatpaks, since they must be built from a + # compose of a single module. group = self.container_build_parser.add_mutually_exclusive_group() group.add_argument( @@ -1405,35 +1450,16 @@ see API KEY section of copr-cli(1) man page. 'Cannot be used with --signing-intent or --compose-id', nargs='+') - self.container_build_parser.add_argument( - '--target', - help='Override the default target', - default=None) - - self.container_build_parser.add_argument( - '--nowait', - action='store_true', - default=False, - help="Don't wait on build") - - self.container_build_parser.add_argument( - '--scratch', - help='Scratch build', - action="store_true") - - self.container_build_parser.add_argument( - '--arches', - action='store', - nargs='*', - help='Limit a scratch build to an arch. May have multiple arches.') + self.container_build_parser.set_defaults(command=self.container_build) - self.container_build_parser.add_argument( - '--skip-remote-rules-validation', - action='store_true', - default=False, - help="Don't check if there's a valid gating.yaml file in the repo") + def register_flatpak_build(self): + self.flatpak_build_parser = self.subparsers.add_parser( + 'flatpak-build', + help='Build a Flatpak', + description='Build a Flatpak', + parents=[self.container_build_parser_common]) - self.container_build_parser.set_defaults(command=self.container_build) + self.flatpak_build_parser.set_defaults(command=self.flatpak_build) def register_container_build_setup(self): self.container_build_setup_parser = \ @@ -1762,7 +1788,7 @@ see API KEY section of copr-cli(1) man page. # Keep it around for backward compatibility self.container_build() - def container_build(self): + def container_build(self, flatpak=False): target_override = False # Override the target if we were supplied one if self.args.target: @@ -1771,11 +1797,15 @@ see API KEY section of copr-cli(1) man page. opts = {"scratch": self.args.scratch, "quiet": self.args.q, - "yum_repourls": self.args.repo_url, "git_branch": self.cmd.branch_merge, - "arches": self.args.arches, + "arches": self.args.arches} + + if not flatpak: + opts.update({ + "yum_repourls": self.args.repo_url, "compose_ids": self.args.compose_ids, - "signing_intent": self.args.signing_intent} + "signing_intent": self.args.signing_intent, + }) section_name = "%s.container-build" % self.name err_msg = "Missing %(option)s option in [%(plugin.section)s] section. " \ @@ -1808,6 +1838,10 @@ see API KEY section of copr-cli(1) man page. self.check_remote_rules_gating() + # We use MBS to find information about the module to build into a Flatpak + if flatpak: + self.set_module_api_url() + self.cmd.container_build_koji( target_override, opts=opts, @@ -1815,7 +1849,11 @@ see API KEY section of copr-cli(1) man page. kojiprofile=kojiprofile, build_client=build_client, koji_task_watcher=koji_cli.lib.watch_tasks, - nowait=self.args.nowait) + nowait=self.args.nowait, + flatpak=flatpak) + + def flatpak_build(self): + self.container_build(flatpak=True) def container_build_setup(self): self.cmd.container_build_setup(get_autorebuild=self.args.get_autorebuild, diff --git a/requirements/fedora-py2.txt b/requirements/fedora-py2.txt index 0cf79de..e97c9c8 100644 --- a/requirements/fedora-py2.txt +++ b/requirements/fedora-py2.txt @@ -1,10 +1,13 @@ +libmodulemd python2-cccolutils python2-GitPython +python2-gobject-base python2-koji python2-pycurl python-six python2-rpm # rpm-python originally python2-requests +PyYAML # python2-openidc-client # used for MBS OIDC authentication # python2-requests-kerberos # used for MBS Kerberos authentication diff --git a/requirements/fedora-py3.txt b/requirements/fedora-py3.txt index cc99912..cb71549 100644 --- a/requirements/fedora-py3.txt +++ b/requirements/fedora-py3.txt @@ -1,10 +1,13 @@ +libmodulemd python3-cccolutils python3-GitPython +python3-gobject-base python3-koji python3-pycurl python3-six python3-rpm # rpm-python originally python3-requests +python3-yaml # python3-openidc-client # used for MBS OIDC authentication # python3-requests-kerberos # used for MBS Kerberos authentication diff --git a/requirements/pypi.txt b/requirements/pypi.txt index 5d35f3b..36559bf 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -6,8 +6,10 @@ cccolutils >= 1.4 GitPython koji >= 1.15 pycurl >= 7.19 +PyGObject requests six >= 1.9.0 +PyYAML # rpm-py-installer # diff --git a/tests/test_cli.py b/tests/test_cli.py index e8c9c42..83f0686 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -223,7 +223,8 @@ class TestContainerBuildWithKoji(CliTestCase): kojiprofile='koji', build_client=utils.build_client, koji_task_watcher=koji_cli.lib.watch_tasks, - nowait=False + nowait=False, + flatpak=False ) def test_override_target(self): @@ -250,7 +251,8 @@ class TestContainerBuildWithKoji(CliTestCase): kojiprofile='koji', build_client=utils.build_client, koji_task_watcher=koji_cli.lib.watch_tasks, - nowait=False + nowait=False, + flatpak=False ) def test_using_deprecated_kojiconfig(self): @@ -286,7 +288,8 @@ class TestContainerBuildWithKoji(CliTestCase): kojiprofile=None, build_client=utils.build_client, koji_task_watcher=koji_cli.lib.watch_tasks, - nowait=False + nowait=False, + flatpak=False ) def test_use_container_build_own_config(self): @@ -304,6 +307,40 @@ class TestContainerBuildWithKoji(CliTestCase): self.assertEqual('koji-container', kwargs['kojiprofile']) self.assertEqual('koji', kwargs['build_client']) + @patch('requests.get') + def test_flatpak(self, mock_get): + mock_rv = Mock() + mock_rv.ok = True + mock_rv.json.return_value = { + 'auth_method': 'oidc', + 'api_version': 2 + } + + mock_get.return_value = mock_rv + + cli_cmd = ['rpkg', '--path', self.cloned_repo_path, + 'flatpak-build'] + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.flatpak_build() + + self.mock_container_build_koji.assert_called_once_with( + False, + opts={ + 'scratch': False, + 'quiet': False, + 'git_branch': 'eng-rhel-7', + 'arches': None, + }, + kojiconfig=None, + kojiprofile='koji', + build_client=utils.build_client, + koji_task_watcher=koji_cli.lib.watch_tasks, + nowait=False, + flatpak=True + ) + class TestClog(CliTestCase): diff --git a/tests/test_flatpak_build.py b/tests/test_flatpak_build.py new file mode 100644 index 0000000..8359149 --- /dev/null +++ b/tests/test_flatpak_build.py @@ -0,0 +1,296 @@ +import os +import subprocess +from textwrap import dedent + +from mock import Mock, patch +import requests + +from utils import CommandTestCase + +EOG_MODULEMD = """ +document: modulemd +version: 2 +data: + name: eog + stream: f28 + version: 20170629213428 + summary: Eye of GNOME Application Module + description: The Eye of GNOME image viewer (eog) is the official image viewer for + the GNOME desktop. It can view single image files in a variety of formats, as + well as large image collections. + license: + module: [MIT] + dependencies: + - buildrequires: + flatpak-runtime: [f28] + requires: + flatpak-runtime: [f28] + profiles: + default: + rpms: [eog] + components: + rpms: {} + xmd: + mbs: OMITTED +""" + +FLATPAK_RUNTIME_MODULEMD = """ +document: modulemd +version: 2 +data: + name: flatpak-runtime + stream: f28 + version: 20170701152209 + summary: Flatpak Runtime + description: Libraries and data files shared between applications + api: + rpms: [librsvg2, gnome-themes-standard, abattis-cantarell-fonts, rest, xkeyboard-config, + adwaita-cursor-theme, python3-gobject-base, json-glib, zenity, gsettings-desktop-schemas, + glib-networking, gobject-introspection, gobject-introspection-devel, flatpak-rpm-macros, + python3-gobject, gvfs-client, colord-libs, flatpak-runtime-config, hunspell-en-GB, + libsoup, glib2-devel, hunspell-en-US, at-spi2-core, gtk3, libXtst, adwaita-gtk2-theme, + libnotify, adwaita-icon-theme, libgcab1, libxkbcommon, libappstream-glib, python3-cairo, + gnome-desktop3, libepoxy, hunspell, libgusb, glib2, enchant, at-spi2-atk] + dependencies: + - buildrequires: + platform: [f28] + requires: + platform: [f28] + license: + module: [MIT] + profiles: + buildroot: + rpms: [flatpak-rpm-macros, flatpak-runtime-config] + runtime: + rpms: [libwayland-server, librsvg2, libX11, libfdisk, adwaita-cursor-theme, + libsmartcols, popt, gdbm, libglvnd, openssl-libs, gobject-introspection, systemd, + ncurses-base, lcms2, libpcap, crypto-policies, fontconfig, libacl, libwayland-cursor, + libseccomp, gmp, jbigkit-libs, bzip2-libs, libunistring, freetype, nettle, + libidn, python3-six, gtk2, gtk3, ca-certificates, libdrm, rest, lzo, libcap, + gnutls, pango, util-linux, basesystem, p11-kit, libgcab1, iptables-libs, dbus, + python3-gobject-base, cryptsetup-libs, krb5-libs, sqlite-libs, kmod-libs, + libmodman, libarchive, enchant, libXfixes, systemd-libs, shared-mime-info, + coreutils-common, libglvnd-glx, abattis-cantarell-fonts, cairo, audit-libs, + libwayland-client, libpciaccess, sed, libgcc, libXrender, json-glib, libxshmfence, + glib-networking, libdb, fedora-modular-repos, keyutils-libs, hwdata, glibc, + libproxy, python3-pyparsing, device-mapper, libgpg-error, system-python, shadow-utils, + libXtst, libstemmer, dbus-libs, libpng, cairo-gobject, libXau, pcre, python3-packaging, + at-spi2-core, gawk, mesa-libglapi, libXinerama, adwaita-gtk2-theme, libX11-common, + device-mapper-libs, python3-appdirs, libXrandr, bash, glibc-common, libselinux, + elfutils-libs, libxkbcommon, libjpeg-turbo, libuuid, atk, acl, libmount, lz4-libs, + ncurses, libgusb, glib2, python3, libpwquality, at-spi2-atk, libattr, libcrypt, + gnome-themes-standard, libtiff, harfbuzz, libstdc++, libXcomposite, xkeyboard-config, + libxcb, libnotify, systemd-pam, readline, libXxf86vm, python3-cairo, gtk-update-icon-cache, + python3-pip, mesa-libEGL, zenity, python3-gobject, libXcursor, tzdata, gvfs-client, + libverto, libblkid, cracklib, libusbx, libcroco, libdatrie, gdk-pixbuf2, libXi, + qrencode-libs, python3-libs, graphite2, mesa-libwayland-egl, mesa-libGL, pixman, + libXext, glibc-all-langpacks, info, grep, fedora-modular-release, setup, zlib, + libtasn1, libepoxy, hunspell, libsemanage, python3-setuptools, fontpackages-filesystem, + libsigsegv, hicolor-icon-theme, libxml2, expat, libgcrypt, emacs-filesystem, + gsettings-desktop-schemas, chkconfig, xz-libs, mesa-libgbm, libthai, coreutils, + colord-libs, libcap-ng, flatpak-runtime-config, elfutils-libelf, hunspell-en-GB, + libsoup, pam, hunspell-en-US, jasper-libs, p11-kit-trust, avahi-libs, elfutils-default-yama-scope, + libutempter, adwaita-icon-theme, ncurses-libs, libidn2, system-python-libs, + libffi, libXdamage, libglvnd-egl, libXft, cups-libs, ustr, libcom_err, libappstream-glib, + gnome-desktop3, gdk-pixbuf2-modules, libsepol, filesystem, gzip, mpfr] + sdk: + rpms: [gcc] + components: + rpms: {} + xmd: + flatpak: + # This gives information about how to map this module into Flatpak terms + # this is used when building application modules against this module. + branch: f28 + runtimes: # Keys are profile names + runtime: + id: org.fedoraproject.Platform + sdk: org.fedoraproject.Sdk + sdk: + id: org.fedoraproject.Sdk + runtime: org.fedoraproject.Platform + mbs: OMITTED +""" # noqa + +UNEXPANDED_MODULEMD = """ +document: modulemd +version: 2 +data: + name: nodeps + stream: f28 + version: 20181234567890 + summary: No dependencies + description: This module has no deps + license: + module: [MIT] + dependencies: + - buildrequires: + platform: [f27, f28] + requires: + platform: [f27, f28] + components: + rpms: {} +""" + +NODEPS_MODULEMD = """ +document: modulemd +version: 2 +data: + name: nodeps + stream: f28 + version: 20181234567890 + summary: No dependencies + description: This module has no deps + license: + module: [MIT] + dependencies: [] + components: + rpms: {} +""" + +BUILDS = { + 'eog:f28': [ + {'modulemd': EOG_MODULEMD} + ], + 'eog:f28:20170629213428': [ + {'modulemd': EOG_MODULEMD} + ], + 'flatpak-runtime:f28': [ + {'modulemd': FLATPAK_RUNTIME_MODULEMD} + ], + 'bad-modulemd:f28': [ + {'modulemd': "BLAH"} + ], + 'unexpanded:f28': [ + {'modulemd': UNEXPANDED_MODULEMD} + ], + 'nodeps:f28': [ + {'modulemd': NODEPS_MODULEMD} + ], +} + + +class FlatpakBuildCase(CommandTestCase): + def set_container_modules(self, container_modules): + with open(os.path.join(self.repo_path, 'container.yaml'), 'w') as f: + f.write(dedent("""\ + compose: + modules: {0} + """.format(container_modules))) + git_cmds = [ + ['git', 'add', 'container.yaml'], + ['git', 'commit', '-m', 'Update container.yaml'], + ] + for cmd in git_cmds: + self.run_cmd(cmd, cwd=self.repo_path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + git_cmds = [ + ['git', 'fetch', 'origin'], + ['git', 'reset', '--hard', 'origin/master'], + ] + for cmd in git_cmds: + self.run_cmd(cmd, cwd=self.cloned_repo_path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def setUp(self): + super(FlatpakBuildCase, self).setUp() + + self.cmd = self.make_commands() + self.cmd.module_api_url = "https://mbs.example.com/module-build-service/1/" + + self.requests_get_p = patch('requests.get') + self.mock_requests_get = self.requests_get_p.start() + + def mock_get(url, params=None, timeout=None): + nsvc = params['nsvc'] + del params['nsvc'] + self.assertEquals(params, { + 'order_desc_by': 'version', + 'per_page': 1 + }) + + response = Mock(requests.Response) + response.ok = True + response.json.return_value = {'items': BUILDS.get(nsvc, [])} + + return response + + self.mock_requests_get.side_effect = mock_get + + self.load_krb_user_p = patch('pyrpkg.Commands._load_krb_user') + self.mock_load_krb_user = self.load_krb_user_p.start() + + session = Mock() + self.kojisession = session + session.system.listMethods.return_value = ['buildContainer'] + + def load_kojisession(self): + self._kojisession = session + + self.load_kojisession_p = patch('pyrpkg.Commands.load_kojisession', + new=load_kojisession) + self.mock_load_kojisession = self.load_kojisession_p.start() + + session.getBuildTarget.return_value = {'dest_tag': 'f28-flatpak'} + session.getTag.return_value = {'locked': False} + + def tearDown(self): + self.requests_get_p.stop() + self.load_krb_user_p.stop() + self.load_kojisession_p.stop() + + super(FlatpakBuildCase, self).tearDown() + + def test_find_target(self): + self.set_container_modules(['eog:f28']) + assert self.cmd.flatpak_build_target == 'f28-flatpak-candidate' + + def test_find_target_version(self): + self.set_container_modules(['eog:f28:20170629213428']) + assert self.cmd.flatpak_build_target == 'f28-flatpak-candidate' + + def module_failure(self, container_modules, exception_str): + if container_modules is not None: + self.set_container_modules(container_modules) + with self.assertRaises(Exception) as e: + self.cmd.load_flatpak_build_target() + self.assertIn(exception_str, str(e.exception)) + + def test_find_target_no_container_yaml(self): + self.module_failure(None, "Cannot find 'container.yaml'") + + def test_find_target_no_modules(self): + self.module_failure([], "No modules listed in 'container.yaml'") + + def test_find_target_multiple_modules(self): + self.module_failure(['eog:f28', 'foo:f28'], + "Multiple modules listed in 'container.yaml'") + + def test_find_target_bad_nsv(self): + self.module_failure(['NOT_A_MODULE'], "should be NAME:STREAM[:VERSION]") + + def test_find_target_no_builds(self): + self.module_failure(['eog:f1'], "Cannot find any builds for module") + + def test_find_target_bad_modulemd(self): + self.module_failure(['bad-modulemd:f28'], "Failed to load modulemd") + + def test_find_target_unexpected(self): + self.module_failure(['unexpanded:f28'], "stream list for 'platform' is not expanded") + + def test_find_target_no_platform(self): + self.module_failure(['nodeps:f28'], "Unable to find 'platform' module in the dependencies") + + def test_flatpak_build(self): + self.set_container_modules(['eog:f28']) + self.cmd.container_build_koji(nowait=True, flatpak=True) + + session = self.kojisession + session.getBuildTarget.assert_called_with('f28-flatpak-candidate') + session.getTag.assert_called_with('f28-flatpak') + + session.buildContainer.assert_called() + args, kwargs = session.buildContainer.call_args + source, container_target, taskinfo = args + + assert container_target == 'f28-flatpak-candidate' From 82f763ccdfed1eb7930203ff5b9b6a08d356c0c9 Mon Sep 17 00:00:00 2001 From: Owen W. Taylor Date: Sep 13 2018 14:58:58 +0000 Subject: [PATCH 3/3] Don't registry flatpak-build command on Python-2.6 libmodulemd is not compatible with Python-2.6 (because of the PyGObject), so a) don't register the flaptak-build command for Python-2.6 and b) skip the related tests. Signed-off-by: Owen W. Taylor --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 26d637a..cc3c6ba 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -48,9 +48,12 @@ from pyrpkg.lookaside import CGILookasideCache from pyrpkg.sources import SourcesFile from pyrpkg.utils import cached_property, log_result, find_me -import gi -gi.require_version('Modulemd', '1.0') -from gi.repository import Modulemd # noqa +PY26 = sys.version_info < (2, 7, 0) + +if not PY26: + import gi + gi.require_version('Modulemd', '1.0') # raises ValueError + from gi.repository import Modulemd # noqa class NullHandler(logging.Handler): @@ -2927,6 +2930,7 @@ class Commands(object): koji_task_watcher=None, nowait=False, flatpak=False): + # check if repo is dirty and all commits are pushed self.check_repo() container_target = self.target if target_override \ diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index 128d516..b0553d6 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -31,7 +31,7 @@ import koji_cli.lib import pyrpkg.utils as utils import six -from pyrpkg import rpkgError, log as rpkgLogger +from pyrpkg import PY26, rpkgError, log as rpkgLogger from six.moves import configparser @@ -439,7 +439,8 @@ class cliClient(object): self.register_container_build() self.register_container_build_setup() self.register_diff() - self.register_flatpak_build() + if not PY26: + self.register_flatpak_build() self.register_gimmespec() self.register_gitbuildhash() self.register_gitcred() diff --git a/requirements/pypi.txt b/requirements/pypi.txt index 36559bf..e4c43bb 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -6,7 +6,6 @@ cccolutils >= 1.4 GitPython koji >= 1.15 pycurl >= 7.19 -PyGObject requests six >= 1.9.0 PyYAML diff --git a/tests/test_cli.py b/tests/test_cli.py index 83f0686..745a97c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,6 +23,7 @@ from six.moves import http_client import git import pyrpkg.cli +from pyrpkg import PY26 try: import openidc_client @@ -307,6 +308,9 @@ class TestContainerBuildWithKoji(CliTestCase): self.assertEqual('koji-container', kwargs['kojiprofile']) self.assertEqual('koji', kwargs['build_client']) + @unittest.skipIf( + PY26, + 'Skip on old Python versions where libmodulemd is not available.') @patch('requests.get') def test_flatpak(self, mock_get): mock_rv = Mock() diff --git a/tests/test_flatpak_build.py b/tests/test_flatpak_build.py index 8359149..97ab456 100644 --- a/tests/test_flatpak_build.py +++ b/tests/test_flatpak_build.py @@ -2,9 +2,15 @@ import os import subprocess from textwrap import dedent +try: + import unittest2 as unittest +except ImportError: + import unittest + from mock import Mock, patch import requests +from pyrpkg import PY26 from utils import CommandTestCase EOG_MODULEMD = """ @@ -170,6 +176,9 @@ BUILDS = { } +@unittest.skipIf( + PY26, + 'Skip on old Python versions where libmodulemd is not available.') class FlatpakBuildCase(CommandTestCase): def set_container_modules(self, container_modules): with open(os.path.join(self.repo_path, 'container.yaml'), 'w') as f: diff --git a/tox.ini b/tox.ini index 707c7c3..da909bf 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,12 @@ envlist = py26,py27,py36,py37,flake8,doc [testenv] skip_install = True -deps = +base_deps = rpm-py-installer -r{toxinidir}/requirements/test-pypi.txt +deps = + {[testenv]base_deps} + PyGObject commands = nosetests {posargs} setenv= @@ -17,7 +20,7 @@ deps = # Since this version, Python 2.6 support has been dropped. pyOpenSSL<18.0.0 unittest2 - {[testenv]deps} + {[testenv]base_deps} [testenv:flake8] basepython = python3