From a48d7f49f150bcc271a62da5c9bab1aea4212662 Mon Sep 17 00:00:00 2001 From: mprahl Date: Oct 12 2017 17:27:04 +0000 Subject: [PATCH 1/2] Add .vscode to .gitignore Signed-off-by: mprahl --- diff --git a/.gitignore b/.gitignore index ff993e6..e0c26c1 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ dist/ rpkg.egg-info/ /sources .coverage +.vscode From f6c308aeb8491efc709004127fe7caf12948549b Mon Sep 17 00:00:00 2001 From: mprahl Date: Oct 17 2017 13:38:58 +0000 Subject: [PATCH 2/2] Port mbs-build to rpkg Signed-off-by: mprahl --- diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py index 5c6c789..527133e 100644 --- a/pyrpkg/__init__.py +++ b/pyrpkg/__init__.py @@ -9,6 +9,7 @@ # option) any later version. See http://www.gnu.org/copyleft/gpl.html for # the full text of the license. +from __future__ import print_function import cccolutils import errno import fnmatch @@ -26,9 +27,14 @@ import six import sys import tempfile import subprocess +import json +import time +from multiprocessing.dummy import Pool as ThreadPool from six.moves import configparser from six.moves import urllib +from six.moves.urllib.parse import urljoin +import requests from pyrpkg.errors import HashtypeMixingError, rpkgError, rpkgAuthError, \ UnknownTargetError @@ -2664,3 +2670,484 @@ class Commands(object): cmd.append('--nowait') cmd.extend([project, srpm_name]) self._run_command(cmd) + + def module_build_cancel(self, api_url, build_id, auth_method, + oidc_id_provider=None, oidc_client_id=None, + oidc_client_secret=None, oidc_scopes=None): + """ + Cancel an MBS build + :param api_url: a string of the URL of the MBS API + :param build_id: an integer of the build ID to cancel + :param auth_method: a string of the authentication method used by the + MBS + :kwarg oidc_id_provider: a string of the OIDC provider when MBS is + using OIDC for authentication + :kwarg oidc_client_id: a string of the OIDC client ID when MBS is + using OIDC for authentication + :kwarg oidc_client_secret: a string of the OIDC client secret when MBS + is using OIDC for authentication. Based on the OIDC setup, this could + be None. + :kwarg oidc_scopes: a list of OIDC scopes when MBS is using OIDC for + authentication + :return: None + """ + # Make sure the build they are trying to cancel exists + self.module_get_build(api_url, build_id) + url = self.module_get_url(api_url, build_id, action='PATCH') + resp = self.module_send_authorized_request( + 'PATCH', url, {'state': 'failed'}, auth_method, oidc_id_provider, + oidc_client_id, oidc_client_secret, oidc_scopes, timeout=60) + if not resp.ok: + try: + error_msg = resp.json()['message'] + except (ValueError, KeyError): + error_msg = resp.text + raise rpkgError( + 'The cancellation of module build #{0} failed with:\n{1}' + .format(build_id, error_msg)) + + def module_build_info(self, api_url, build_id): + """ + Show information about an MBS build + :param api_url: a string of the URL of the MBS API + :param build_id: an integer of the build ID to query MBS about + :return: None + """ + # Load the Koji session anonymously so we get access to the Koji web + # URL + self.load_kojisession(anon=True) + state_names = self.module_get_koji_state_dict() + data = self.module_get_build(api_url, build_id) + print('Name: {0}'.format(data['name'])) + print('Stream: {0}'.format(data['stream'])) + print('Version: {0}'.format(data['version'])) + print('Koji Tag: {0}'.format(data['koji_tag'])) + print('Owner: {0}'.format(data['owner'])) + print('State: {0}'.format(data['state_name'])) + print('State Reason: {0}'.format(data['state_reason'] or '')) + print('Time Submitted: {0}'.format(data['time_submitted'])) + print('Time Completed: {0}'.format(data['time_completed'])) + print('Components:') + for package_name, task_data in data['tasks'].get('rpms', {}).items(): + koji_task_url = '' + if task_data.get('task_id'): + koji_task_url = '{0}/taskinfo?taskID={1}'.format( + self.kojiweburl, task_data['task_id']) + print(' Name: {0}'.format(package_name)) + print(' NVR: {0}'.format(task_data['nvr'])) + print(' State: {0}'.format( + state_names[task_data.get('state', None)])) + print(' Koji Task: {0}\n'.format(koji_task_url)) + + def module_get_build(self, api_url, build_id): + """ + Get an MBS build + :param api_url: a string of the URL of the MBS API + :param build_id: an integer of the build ID to query MBS about + :return: None or a dictionary representing the module build + """ + url = self.module_get_url(api_url, build_id) + response = requests.get(url, timeout=60) + if response.ok: + return response.json() + else: + try: + error_msg = response.json()['message'] + except (ValueError, KeyError): + error_msg = response.text + raise rpkgError( + 'The following error occurred while getting information on ' + 'module build #{0}:\n{1}'.format(build_id, error_msg)) + + def module_get_url(self, api_url, build_id, action='GET'): + """ + Get the proper MBS API URL for the desired action + :param api_url: a string of the URL of the MBS API + :param build_id: an integer of the module build desired. If this is set + to None, then the base URL for all module builds is returned. + :kwarg action: a string determining the HTTP action. If this is set to + GET, then the URL will contain `?verbose=true`. Any other value will + not have verbose set. + :return: a string of the desired MBS API URL + """ + url = urljoin(api_url, 'module-builds/') + if build_id is not None: + url = '{0}{1}'.format(url, build_id) + else: + url = '{0}'.format(url) + + if action == 'GET': + url = '{0}?verbose=true'.format(url) + return url + + @staticmethod + def module_get_koji_state_dict(): + """ + Get a dictionary of Koji build states with the keys being strings and + the values being their associated integer + :return: a dictionary of Koji build states + """ + state_names = dict([(v, k) for k, v in koji.BUILD_STATES.items()]) + state_names[None] = 'undefined' + return state_names + + def module_get_scm_info(self, scm_url=None, branch=None): + """ + Determines the proper SCM URL and branch based on the arguments. If the + user doesn't specify an SCM URL and branch, then the git repo the user + is currently in is used instead. + :kwarg scm_url: a string of the module's SCM URL + :kwarg branch: a string of the module's branch + :return: a tuple containing a string of the SCM URL and a string of the + branch + """ + if not scm_url: + # Make sure the local repo is clean (no unpushed changes) if the + # user didn't specify an SCM URL + self.check_repo() + + if branch: + actual_branch = branch + else: + # If the branch wasn't specified, make sure they also didn't + # specify an scm_url + if scm_url: + raise rpkgError('You need to specify a branch if you specify ' + 'the SCM URL') + # If the scm_url was not specified, then just use the active + # branch + actual_branch = self.repo.active_branch.name + + if scm_url: + actual_scm_url = scm_url + else: + # If the scm_url isn't specified, get the remote git URL of the + # git repo the current user is in + actual_scm_url = self._get_namespace_anongiturl( + self.ns_module_name) + actual_scm_url = '{0}?#{1}'.format(actual_scm_url, self.commithash) + return actual_scm_url, actual_branch + + def module_local_build(self, scm_url, branch, local_builds_nsvs=None, + skip_tests=False, verbose=False, debug=False): + """ + A wrapper for `mbs-manager build_module_locally`. + :param scm_url: a string of the module's SCM URL. + :param branch: a string of the module's branch. + :kwarg local_builds_nsvs: a list of localbuilds to import into MBS + before running this local build. + :kwarg skip_tests: a boolean determining if the check sections should + be skipped. + :kwarg verbose: a boolean specifying if mbs-manager should be verbose. + This is overridden by self.quiet. + :kwarg debug: a boolean specifying if mbs-manager should be debug. + This is overridden by self.quiet and verbose. + :return: None + """ + command = ['mbs-manager'] + if self.quiet: + command.append('-q') + elif verbose: + command.append('-v') + elif debug: + command.append('-d') + command.append('build_module_locally') + if skip_tests: + command.append('--skiptests') + + if local_builds_nsvs: + for build_id in local_builds_nsvs: + command += ['--add-local-build', build_id] + + command.extend([scm_url, branch]) + self._run_command(command) + + def module_overview(self, api_url, limit=10, finished=True): + """ + Show the overview of the latest builds in MBS + :param api_url: a string of the URL of the MBS API + :kwarg limit: an integer of the number of most recent module builds to + display. This defaults to 10. + :kwarg finished: a boolean that determines if only finished or + unfinished module builds should be displayed. This defaults to True. + :return: None + """ + # Don't let the user cause problems by specifying a negative limit + if limit < 1: + limit = 1 + build_states = { + 'init': 0, + 'wait': 1, + 'build': 2, + 'done': 3, + 'failed': 4, + 'ready': 5, + } + baseurl = self.module_get_url(api_url, build_id=None) + if finished: + # These are the states when a build is finished + states = [build_states['done'], build_states['ready'], + build_states['failed']] + else: + # These are the states when a build is in progress + states = [build_states['init'], build_states['wait'], + build_states['build']] + + def _get_module_builds(state): + """ + Private function that is used for multithreading later on to get + the desired amount of builds for a specific state. + :param state: an integer representing the build state to query for + :return: yields dictionaries of the builds found + """ + total = 0 + page = 1 + # If the limit is above 100, we don't want the amount of results + # per_page to exceed 100 since this is not allowed. + per_page = min(limit, 100) + params = { + 'state': state, + # Order by the latest builds first + 'order_desc_by': 'id', + 'verbose': True, + 'per_page': per_page + } + while total < limit: + params['page'] = page + response = requests.get(baseurl, params=params, timeout=30) + if not response.ok: + try: + error = response.json()['message'] + except (ValueError, KeyError): + error = response.text + raise rpkgError( + 'The request to "{0}" failed with parameters "{1}". ' + 'The status code was "{2}". The error was: {3}' + .format(baseurl, str(params), response.status_code, + error)) + + data = response.json() + for item in data['items']: + total += 1 + yield item + + if data['meta']['next']: + page += 1 + else: + # Even if we haven't reached the desired amount of builds, + # we must break out of the loop because we are out of pages + # to search + break + + # Make this one thread per state we want to query + pool = ThreadPool(3) + # Eventually, the MBS should support a range of states but for now, we + # have to be somewhat wasteful and query per state + module_builds = pool.map( + lambda x: list(_get_module_builds(state=x)), states) + # Make one flat list with all the modules + module_builds = [item for sublist in module_builds for item in sublist] + # Sort the list of builds to be oldest to newest + module_builds.sort(key=lambda x: x['id']) + # Only grab the desired limit starting from the newest builds + module_builds = module_builds[(limit * -1):] + # Track potential duplicates if the state changed in the middle of the + # query + module_build_ids = set() + for build in module_builds: + if build['id'] in module_build_ids: + continue + module_build_ids.add(build['id']) + print('ID: {0}'.format(build['id'])) + print('Name: {0}'.format(build['name'])) + print('Stream: {0}'.format(build['stream'])) + print('Version: {0}'.format(build['version'])) + print('Koji Tag: {0}'.format(build['koji_tag'])) + print('Owner: {0}'.format(build['owner'])) + print('State: {0}\n'.format(build['state_name'])) + + def module_send_authorized_request(self, verb, url, body, auth_method, + oidc_id_provider=None, + oidc_client_id=None, + oidc_client_secret=None, + oidc_scopes=None, **kwargs): + """ + Sends authorized request to MBS + :param verb: a string of the HTTP verb of the request (e.g. POST) + :param url: a string of the URL to make the request on + :param body: a dictionary of the data to send in the authorized request + :param auth_method: a string of the authentication method used by the + MBS + :kwarg oidc_id_provider: a string of the OIDC provider when MBS is + using OIDC for authentication + :kwarg oidc_client_id: a string of the OIDC client ID when MBS is + using OIDC for authentication + :kwarg oidc_client_secret: a string of the OIDC client secret when MBS + is using OIDC for authentication. Based on the OIDC setup, this could + be None. + :kwarg oidc_scopes: a list of OIDC scopes when MBS is using OIDC for + authentication + :kwarg **kwargs: any additional python-requests keyword arguments + :return: a python-requests response object + """ + if auth_method == 'oidc': + import openidc_client + if oidc_id_provider is None or oidc_client_id is None or \ + oidc_scopes is None: + raise ValueError('The selected authentication method was ' + '"oidc" but the OIDC configuration keyword ' + 'arguments were not specified') + + mapping = {'Token': 'Token', 'Authorization': 'Authorization'} + # Get the auth token using the OpenID client + oidc = openidc_client.OpenIDCClient( + 'mbs_build', oidc_id_provider, mapping, oidc_client_id, + oidc_client_secret) + + resp = oidc.send_request( + url, http_method=verb.upper(), json=body, scopes=oidc_scopes, + **kwargs) + elif auth_method == 'kerberos': + import requests_kerberos + + if type(body) is dict: + data = json.dumps(body) + else: + data = body + auth = requests_kerberos.HTTPKerberosAuth( + mutual_authentication=requests_kerberos.OPTIONAL) + resp = requests.request(verb, url, data=data, auth=auth, **kwargs) + if resp.status_code == 401: + raise rpkgError('MBS authentication using Kerberos failed. ' + 'Make sure you have a valid Kerberos ticket.') + else: + # This scenario should not be reached because the config was + # validated in the function that calls this function + raise rpkgError('An unsupported MBS "auth_method" was provided') + return resp + + def module_submit_build(self, api_url, scm_url, branch, auth_method, + optional=None, oidc_id_provider=None, + oidc_client_id=None, oidc_client_secret=None, + oidc_scopes=None): + """ + Submit a module build to the MBS + :param api_url: a string of the URL of the MBS API + :param scm_url: a string of the module's SCM URL + :param branch: a string of the module's branch + :param auth_method: a string of the authentication method used by the + MBS + :param optional: an optional list of "key=value" to be passed in with + the MBS build submission + :kwarg oidc_id_provider: a string of the OIDC provider when MBS is + using OIDC for authentication + :kwarg oidc_client_id: a string of the OIDC client ID when MBS is + using OIDC for authentication + :kwarg oidc_client_secret: a string of the OIDC client secret when MBS + is using OIDC for authentication. Based on the OIDC setup, this could + be None. + :kwarg oidc_scopes: a list of OIDC scopes when MBS is using OIDC for + authentication + :return: None + """ + body = {'scmurl': scm_url, 'branch': branch} + optional = optional if optional else [] + optional_dict = {} + try: + for x in optional: + key, value = x.split('=', 1) + optional_dict[key] = value + except (IndexError, ValueError): + raise rpkgError( + 'Optional arguments are not in the proper "key=value" format') + + body.update(optional_dict) + url = self.module_get_url(api_url, build_id=None, action='POST') + resp = self.module_send_authorized_request( + 'POST', url, body, auth_method, oidc_id_provider, oidc_client_id, + oidc_client_secret, oidc_scopes, timeout=120) + + data = {} + try: + data = resp.json() + return data['id'] + except (KeyError, ValueError): + if 'message' in data: + error_msg = data['message'] + else: + error_msg = resp.text + raise rpkgError('The build failed with:\n{0}'.format(error_msg)) + + def module_watch_build(self, api_url, build_id): + """ + Watches the MBS build in a loop that updates every 15 seconds. + The loop ends when the build state is 'failed', 'done', or 'ready'. + :param api_url: a string of the URL of the MBS API + :param build_id: an integer of the module build to watch. + :return: None + """ + # Load the Koji session anonymously so we get access to the Koji web + # URL + self.load_kojisession(anon=True) + done = False + while not done: + state_names = self.module_get_koji_state_dict() + build = self.module_get_build(api_url, build_id) + tasks = {} + if 'rpms' in build['tasks']: + tasks = build['tasks']['rpms'] + + states = list(set([task['state'] for task in tasks.values()])) + inverted = {} + for name, task in tasks.items(): + state = task['state'] + inverted[state] = inverted.get(state, []) + inverted[state].append(name) + + # Clear the screen + try: + os.system('clear') + except Exception: + # If for whatever reason the clear command fails, fall back to + # clearing the screen using print + print(chr(27) + "[2J") + + # Display all RPMs that have built or have failed + build_state = 0 + failed_state = 3 + for state in (build_state, failed_state): + if state not in inverted: + continue + if state == build_state: + print('Still Building:') + else: + print('Failed:') + for name in inverted[state]: + task = tasks[name] + if task['task_id']: + print(' {0} {1}/taskinfo?taskID={2}'.format( + name, self.kojiweburl, task['task_id'])) + else: + print(' {0}'.format(name)) + + print('\nSummary:') + for state in states: + num_in_state = len(inverted[state]) + if num_in_state == 1: + component_text = 'component' + else: + component_text = 'components' + print(' {0} {1} in the "{2}" state'.format( + num_in_state, component_text, state_names[state].lower())) + + done = build['state_name'] in ['failed', 'done', 'ready'] + + template = ('{owner}\'s build #{id} of {name}-{stream} is in ' + 'the "{state_name}" state') + if build['state_reason']: + template += ' (reason: {state_reason})' + if build.get('koji_tag'): + template += ' (koji tag: "{koji_tag}")' + print(template.format(**build)) + if not done: + time.sleep(15) diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py index ac72148..24c97d1 100644 --- a/pyrpkg/cli.py +++ b/pyrpkg/cli.py @@ -12,6 +12,7 @@ # There are 6 functions derived from /usr/bin/koji which are licensed under # LGPLv2.1. See comments before those functions. +from __future__ import print_function import argparse import logging import os @@ -290,6 +291,12 @@ class cliClient(object): self.register_local() self.register_mockbuild() self.register_mock_config() + self.register_module_build() + self.register_module_build_cancel() + self.register_module_build_info() + self.register_module_local_build() + self.register_module_build_watch() + self.register_module_overview() self.register_new() self.register_new_sources() self.register_patch() @@ -705,6 +712,87 @@ defined, packages will be built sequentially.""" % {'name': self.name}) mock_config_parser.add_argument('--arch', help='Override local arch') mock_config_parser.set_defaults(command=self.mock_config) + def register_module_build(self): + sub_help = 'Build a module using MBS' + self.module_build_parser = self.subparsers.add_parser( + 'module-build', help=sub_help, description=sub_help) + self.module_build_parser.add_argument( + 'scm_url', nargs='?', + help='The module\'s SCM URL. This defaults to the current repo.') + self.module_build_parser.add_argument( + 'branch', nargs='?', + help=('The module\'s SCM branch. This defaults to the current ' + 'checked-out branch.')) + self.module_build_parser.add_argument( + '--watch', '-w', help='Watch the module build', + action='store_true') + self.module_build_parser.add_argument( + '--optional', action='append', metavar='KEY=VALUE', + dest='optional', + help='MBS optional arguments in the form of "key=value"') + self.module_build_parser.set_defaults(command=self.module_build) + + def register_module_build_cancel(self): + sub_help = 'Cancel an MBS module build' + self.module_build_cancel_parser = self.subparsers.add_parser( + 'module-build-cancel', help=sub_help, description=sub_help) + self.module_build_cancel_parser.add_argument( + 'build_id', help='The ID of the module build to cancel', type=int) + self.module_build_cancel_parser.set_defaults( + command=self.module_build_cancel) + + def register_module_build_info(self): + sub_help = 'Show information of an MBS module build' + self.module_build_info_parser = self.subparsers.add_parser( + 'module-build-info', help=sub_help, description=sub_help) + self.module_build_info_parser.add_argument( + 'build_id', help='The ID of the module build', type=int) + self.module_build_info_parser.set_defaults( + command=self.module_build_info) + + def register_module_local_build(self): + sub_help = 'Build a module locally using the mbs-manager command' + self.module_build_local_parser = self.subparsers.add_parser( + 'module-build-local', help=sub_help, description=sub_help) + self.module_build_local_parser.add_argument( + 'scm_url', nargs='?', + help='The module\'s SCM URL. This defaults to the current repo.') + self.module_build_local_parser.add_argument( + 'branch', nargs='?', + help=('The module\'s SCM branch. This defaults to the current ' + 'checked-out branch.')) + self.module_build_local_parser.add_argument( + '--skip-tests', help='Adds a macro for skipping the check section', + action='store_true') + self.module_build_local_parser.add_argument( + '--add-local-build', action='append', dest='local_builds_nsvs', + metavar='BUILD_ID', type=int, + help='Import previously finished local module builds into MBS') + self.module_build_local_parser.set_defaults( + command=self.module_build_local) + + def register_module_build_watch(self): + sub_help = 'Watch an MBS build' + self.module_build_watch_parser = self.subparsers.add_parser( + 'module-build-watch', help=sub_help, description=sub_help) + self.module_build_watch_parser.add_argument( + 'build_id', help='The ID of the module build to watch', type=int) + self.module_build_watch_parser.set_defaults( + command=self.module_build_watch) + + def register_module_overview(self): + sub_help = 'Shows an overview of MBS builds' + self.module_overview_parser = self.subparsers.add_parser( + 'module-overview', help=sub_help, description=sub_help) + self.module_overview_parser.add_argument( + '--unfinished', help='Show unfinished module builds', + default=False, action='store_true') + self.module_overview_parser.add_argument( + '--limit', default=10, type=int, + help='The number of most recent module builds to display') + self.module_overview_parser.set_defaults( + command=self.module_overview) + def register_new_sources(self): """Register the new-sources target""" @@ -1315,6 +1403,170 @@ see API KEY section of copr-cli(1) man page. def mock_config(self): print(self.cmd.mock_config(self.args.target, self.args.arch)) + def module_build(self): + """ + Builds a module using MBS + :return: None + """ + self.module_validate_config() + scm_url, branch = self.cmd.module_get_scm_info( + self.args.scm_url, self.args.branch) + api_url = self.config.get(self.config_section, 'api_url') + auth_method, oidc_id_provider, oidc_client_id, oidc_client_secret, \ + oidc_scopes = self.module_get_auth_config() + + if not self.args.q: + print('Submitting the module build...') + build_id = self._cmd.module_submit_build( + api_url, scm_url, branch, auth_method, self.args.optional, + oidc_id_provider, oidc_client_id, oidc_client_secret, oidc_scopes) + if self.args.watch: + self.module_watch_build(build_id) + elif not self.args.q: + print('The build #{0} was submitted to the MBS' + .format(build_id)) + + def module_build_cancel(self): + """ + Cancel an MBS build + :return: None + """ + self.module_validate_config() + build_id = self.args.build_id + api_url = self.config.get(self.config_section, 'api_url') + auth_method, oidc_id_provider, oidc_client_id, oidc_client_secret, \ + oidc_scopes = self.module_get_auth_config() + + if not self.args.q: + print('Cancelling module build #{0}...'.format(build_id)) + self.cmd.module_build_cancel( + api_url, build_id, auth_method, oidc_id_provider, oidc_client_id, + oidc_client_secret, oidc_scopes) + if not self.args.q: + print('The module build #{0} was cancelled'.format(build_id)) + + def module_build_info(self): + """ + Show information about an MBS build + :return: None + """ + self.module_validate_config() + api_url = self.config.get(self.config_section, 'api_url') + self.cmd.module_build_info(api_url, self.args.build_id) + + def module_build_local(self): + """ + Build a module locally using mbs-manager + :return: None + """ + self.module_validate_config() + scm_url, branch = self.cmd.module_get_scm_info( + self.args.scm_url, self.args.branch) + self.cmd.module_local_build( + scm_url, branch, self.args.local_builds_nsvs, + self.args.skip_tests, verbose=self.args.v, debug=self.args.debug) + + def module_get_auth_config(self): + """ + Get the authentication configuration for the MBS + :return: a tuple consisting of the authentication method, the OIDC ID + provider, the OIDC client ID, the OIDC client secret, and the OIDC + scopes. If the authentication method is not OIDC, the OIDC values in + the tuple are set to None. + """ + auth_method = self.config.get(self.config_section, 'auth_method') + oidc_id_provider = None + oidc_client_id = None + oidc_client_secret = None + oidc_scopes = None + if auth_method == 'oidc': + oidc_id_provider = self.config.get( + self.config_section, 'oidc_id_provider') + oidc_client_id = self.config.get( + self.config_section, 'oidc_client_id') + oidc_scopes_str = self.config.get( + self.config_section, 'oidc_scopes') + oidc_scopes = [ + scope.strip() for scope in oidc_scopes_str.split(',')] + if self.config.has_option(self.config_section, + 'oidc_client_secret'): + oidc_client_secret = self.config.get( + self.config_section, 'oidc_client_secret') + return (auth_method, oidc_id_provider, oidc_client_id, + oidc_client_secret, oidc_scopes) + + def module_build_watch(self): + """ + Watch an MBS build from the command-line + :return: None + """ + self.module_validate_config() + self.module_watch_build(self.args.build_id) + + def module_overview(self): + """ + Show the overview of the latest builds in the MBS + :return: None + """ + self.module_validate_config() + api_url = self.config.get(self.config_section, 'api_url') + self.cmd.module_overview( + api_url, self.args.limit, finished=(not self.args.unfinished)) + + def module_validate_config(self): + """ + Validates the configuration needed for MBS commands + :return: None or rpkgError + """ + self.config_section = '{0}.mbs'.format(self.name) + # Verify that all necessary config options are set + config_error = ('The config option "{0}" in the "{1}" section is ' + 'required') + if not self.config.has_option(self.config_section, 'auth_method'): + raise rpkgError(config_error.format( + 'auth_method', self.config_section)) + required_configs = ['api_url'] + auth_method = self.config.get(self.config_section, 'auth_method') + if auth_method not in ['oidc', 'kerberos']: + raise rpkgError('The MBS authentication mechanism of "{0}" is not ' + 'supported'.format(auth_method)) + + if auth_method == 'oidc': + # Try to import this now so the user gets immediate feedback if + # it isn't installed + try: + import openidc_client # noqa: F401 + except ImportError: + raise rpkgError('python-openidc-client needs to be installed') + required_configs.append('oidc_id_provider') + required_configs.append('oidc_client_id') + required_configs.append('oidc_scopes') + elif auth_method == 'kerberos': + # Try to import this now so the user gets immediate feedback if + # it isn't installed + try: + import requests_kerberos # noqa: F401 + except ImportError: + raise rpkgError( + 'python-requests-kerberos needs to be installed') + + for required_config in required_configs: + if not self.config.has_option(self.config_section, + required_config): + raise rpkgError(config_error.format( + required_config, self.config_section)) + + def module_watch_build(self, build_id): + """ + Watches the MBS build in a loop that updates every 15 seconds. + The loop ends when the build state is 'failed', 'done', or 'ready'. + :param build_id: an integer of the module build to watch + :return: None + """ + self.module_validate_config() + api_url = self.config.get(self.config_section, 'api_url') + self.cmd.module_watch_build(api_url, build_id) + def new(self): new_diff = self.cmd.new() # When running rpkg with old version GitPython<1.0 which returns string diff --git a/requirements/fedora-cli-tools.txt b/requirements/fedora-cli-tools.txt index 7922817..5209cc7 100644 --- a/requirements/fedora-cli-tools.txt +++ b/requirements/fedora-cli-tools.txt @@ -4,3 +4,4 @@ copr-cli mock rpm-build rpmlint +module-build-service # needed for the module-buld-local command diff --git a/requirements/fedora-py2.txt b/requirements/fedora-py2.txt index 83cbf3e..0cf79de 100644 --- a/requirements/fedora-py2.txt +++ b/requirements/fedora-py2.txt @@ -4,6 +4,9 @@ python2-koji python2-pycurl python-six python2-rpm # rpm-python originally +python2-requests +# python2-openidc-client # used for MBS OIDC authentication +# python2-requests-kerberos # used for MBS Kerberos authentication # For running tests python2-coverage @@ -11,3 +14,4 @@ python2-flake8 python2-mock python2-nose python2-rpmfluff +python2-openidc-client # used in MBS tests diff --git a/requirements/fedora-py3.txt b/requirements/fedora-py3.txt index d03efa8..cc99912 100644 --- a/requirements/fedora-py3.txt +++ b/requirements/fedora-py3.txt @@ -4,6 +4,9 @@ python3-koji python3-pycurl python3-six python3-rpm # rpm-python originally +python3-requests +# python3-openidc-client # used for MBS OIDC authentication +# python3-requests-kerberos # used for MBS Kerberos authentication # For running tests python3-coverage @@ -11,3 +14,4 @@ python3-flake8 python3-mock python3-nose python3-rpmfluff +python3-openidc-client # used in MBS tests diff --git a/requirements/pypi.txt b/requirements/pypi.txt index bc1e56a..b307f6f 100644 --- a/requirements/pypi.txt +++ b/requirements/pypi.txt @@ -4,6 +4,10 @@ cccolutils >= 1.4 GitPython >= 0.2.0 pycurl >= 7.43 six >= 1.9.0 +requests +# openidc-client # used for MBS OIDC authentication +# requests-kerberos # used for MBS Kerberos authentication + # Only required for <= Python 2.6 argparse == 1.4.0 @@ -14,6 +18,7 @@ flake8 >= 2.5.5 mock >= 2.0.0 nose >= 1.3.7 git+https://pagure.io/rpmfluff.git@0.5.1#egg=rpmfluff +openidc-client # used in MBS tests # Several package that are not available in PyPI are also listed here. # If rpkg runs from a Python virtualenv, you may need --site-packages to create diff --git a/tests/fixtures/rpkg.conf b/tests/fixtures/rpkg.conf index eedc2c7..ae8e953 100644 --- a/tests/fixtures/rpkg.conf +++ b/tests/fixtures/rpkg.conf @@ -9,3 +9,11 @@ kojiprofile = koji build_client = koji clone_config = bz.default-component %(module)s + +[rpkg.mbs] +auth_method = oidc +api_url = https://mbs.fedoraproject.org/module-build-service/1/ +oidc_id_provider = https://id.fedoraproject.org/openidc/ +oidc_client_id = mbs-authorizer +oidc_client_secret = notsecret +oidc_scopes = openid,https://id.fedoraproject.org/scope/groups,https://mbs.fedoraproject.org/oidc/submit-build diff --git a/tests/test_cli.py b/tests/test_cli.py index 598ec4c..a193eb3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,13 +24,11 @@ from six.moves import StringIO import git import pyrpkg.cli +import openidc_client import utils -from mock import PropertyMock -from mock import call -from mock import mock_open -from mock import patch -from pyrpkg import rpkgError +from mock import PropertyMock, call, mock_open, patch, Mock +from pyrpkg import rpkgError, Commands from utils import CommandTestCase @@ -1705,3 +1703,429 @@ class TestPatch(CliTestCase): exists.assert_called_once_with( os.path.join(cli.cmd.path, patch_file)) + + +class TestModulesCli(CliTestCase): + """Test module commands""" + + scopes = [ + 'openid', + 'https://id.fedoraproject.org/scope/groups', + 'https://mbs.fedoraproject.org/oidc/submit-build' + ] + module_build_json = { + 'component_builds': [ + 59417, 59418, 59419, 59420, 59421, 59422, 59423, 59428, + 59424, 59425], + 'id': 2150, + 'koji_tag': 'module-14050f52e62d955b', + 'modulemd': '...', + 'name': 'python3-ecosystem', + 'owner': 'torsava', + 'scmurl': ('git://pkgs.fedoraproject.org/modules/python3-ecosystem' + '?#34774a9416c799aadda74f2c44ec4dba4d519c04'), + 'state': 4, + 'state_name': 'failed', + 'state_reason': 'Some error', + 'state_trace': [], + 'state_url': '/module-build-service/1/module-builds/1093', + 'stream': 'master', + 'tasks': { + 'rpms': { + 'module-build-macros': { + 'nvr': 'module-build-macros-None-None', + 'state': 3, + 'state_reason': 'Some error', + 'task_id': 22370514 + }, + 'python-cryptography': { + 'nvr': None, + 'state': 3, + 'state_reason': 'Some error', + 'task_id': None + }, + 'python-dns': { + 'nvr': None, + 'state': 3, + 'state_reason': 'Some error', + 'task_id': None + } + } + }, + 'time_completed': '2017-10-11T09:42:11Z', + 'time_modified': '2017-10-11T09:42:11Z', + 'time_submitted': '2017-10-10T14:55:33Z', + 'version': '20171010145511' + } + + @patch('sys.stdout', new=StringIO()) + @patch.object(openidc_client.OpenIDCClient, 'send_request') + def test_module_build(self, mock_oidc_req): + """ + Test a module build with an SCM URL and branch supplied + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build', + 'git://pkgs.fedoraproject.org/modules/testmodule?#79d87a5a', + 'master' + ] + mock_rv = Mock() + mock_rv.json.return_value = {'id': 1094} + mock_oidc_req.return_value = mock_rv + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_build() + + exp_url = ('https://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/') + exp_json = { + 'scmurl': ('git://pkgs.fedoraproject.org/modules/testmodule?' + '#79d87a5a'), + 'branch': 'master'} + mock_oidc_req.assert_called_once_with( + exp_url, + http_method='POST', + json=exp_json, + scopes=self.scopes, + timeout=120) + output = sys.stdout.getvalue().strip() + expected_output = ('Submitting the module build...\nThe build #1094 ' + 'was submitted to the MBS') + self.assertEqual(output, expected_output) + + @patch('sys.stdout', new=StringIO()) + @patch.object(openidc_client.OpenIDCClient, 'send_request') + def test_module_build_input(self, mock_oidc_req): + """ + Test a module build with default parameters + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build' + ] + mock_rv = Mock() + mock_rv.json.return_value = {'id': 1094} + mock_oidc_req.return_value = mock_rv + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_build() + + output = sys.stdout.getvalue().strip() + expected_output = ('Submitting the module build...\nThe build #1094 ' + 'was submitted to the MBS') + self.assertEqual(output, expected_output) + # Can't verify the calls since the SCM commit hash always changes + mock_oidc_req.assert_called_once() + + @patch('sys.stdout', new=StringIO()) + @patch('requests.get') + @patch.object(openidc_client.OpenIDCClient, 'send_request') + def test_module_cancel(self, mock_oidc_req, mock_get): + """ + Test canceling a module build when the build exists + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build-cancel', + '1125' + ] + mock_rv = Mock() + mock_rv.json.return_value = {'id': 1094} + mock_get.return_value = mock_rv + mock_rv_two = Mock() + mock_rv_two.json.ok = True + mock_oidc_req.return_value = mock_rv_two + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_build_cancel() + exp_url = ('https://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/1125?verbose=true') + mock_get.assert_called_once_with(exp_url, timeout=60) + exp_url_two = ('https://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/1125') + mock_oidc_req.assert_called_once_with( + exp_url_two, + http_method='PATCH', + json={'state': 'failed'}, + scopes=self.scopes, + timeout=60) + output = sys.stdout.getvalue().strip() + expected_output = ('Cancelling module build #1125...\nThe module ' + 'build #1125 was cancelled') + self.assertEqual(output, expected_output) + + @patch('requests.get') + @patch.object(openidc_client.OpenIDCClient, 'send_request') + def test_module_cancel_not_found(self, mock_oidc_req, mock_get): + """ + Test canceling a module build when the build doesn't exist + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build-cancel', + '1125' + ] + mock_rv = Mock() + mock_rv.ok = False + mock_rv.json.return_value = { + 'status': 404, + 'message': 'No such module found.', + 'error': 'Not Found' + } + mock_get.return_value = mock_rv + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + try: + cli.module_build_cancel() + raise RuntimeError('An rpkgError was not raised') + except rpkgError as error: + expected_error = ('The following error occurred while getting ' + 'information on module build #1125:\nNo ' + 'such module found.') + self.assertEqual(str(error), expected_error) + exp_url = ('https://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/1125?verbose=true') + mock_get.assert_called_once_with(exp_url, timeout=60) + mock_oidc_req.assert_not_called() + + @patch('sys.stdout', new=StringIO()) + @patch('requests.get') + def test_module_build_info(self, mock_get): + """ + Test getting information on a module build + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build-info', + '2150' + ] + mock_rv = Mock() + mock_rv.ok = True + mock_rv.json.return_value = self.module_build_json + mock_get.return_value = mock_rv + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_build_info() + exp_url = ('https://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/2150?verbose=true') + mock_get.assert_called_once_with(exp_url, timeout=60) + output = sys.stdout.getvalue().strip() + expected_output = """ +Name: python3-ecosystem +Stream: master +Version: 20171010145511 +Koji Tag: module-14050f52e62d955b +Owner: torsava +State: failed +State Reason: Some error +Time Submitted: 2017-10-10T14:55:33Z +Time Completed: 2017-10-11T09:42:11Z +Components: + Name: module-build-macros + NVR: module-build-macros-None-None + State: FAILED + Koji Task: https://koji.fedoraproject.org/koji/taskinfo?taskID=22370514 + + Name: python-dns + NVR: None + State: FAILED + Koji Task: + + Name: python-cryptography + NVR: None + State: FAILED + Koji Task: +""".strip() # noqa: W291 + self.assertEqual(expected_output, output) + + @patch('sys.stdout', new=StringIO()) + @patch.object(Commands, 'kojiweburl', + 'https://koji.fedoraproject.org/koji') + @patch('requests.get') + @patch('os.system') + @patch.object(Commands, 'load_kojisession') + def test_module_build_watch(self, mock_load_koji, mock_system, mock_get): + """ + Test watching a module build that is already complete + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build-watch', + '1500' + ] + mock_rv = Mock() + mock_rv.ok = True + mock_rv.json.return_value = self.module_build_json + mock_get.return_value = mock_rv + + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_build_watch() + + exp_url = ('https://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/1500?verbose=true') + mock_get.assert_called_once_with(exp_url, timeout=60) + mock_system.assert_called_once_with('clear') + output = sys.stdout.getvalue().strip() + expected_output = """ +Failed: + module-build-macros https://koji.fedoraproject.org/koji/taskinfo?taskID=22370514 + python-dns + python-cryptography + +Summary: + 3 components in the "failed" state +torsava's build #2150 of python3-ecosystem-master is in the "failed" state (reason: Some error) (koji tag: "module-14050f52e62d955b") +""".strip() # noqa: E501 + self.assertEqual(output, expected_output) + + @patch('sys.stdout', new=StringIO()) + @patch('requests.get') + def test_module_overview(self, mock_get): + """ + Test the module overview command with 4 modules in the finished state + and a desired limit of 2 + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-overview', + '--limit', + '2' + ] + # Minimum amount of JSON for the command to succeed + json_one = { + 'items': [], + 'meta': { + 'next': None + } + } + json_two = { + 'items': [ + { + 'id': 1100, + 'koji_tag': 'module-c24f55c24c8fede1', + 'name': 'testmodule', + 'owner': 'jkaluza', + 'state_name': 'ready', + 'stream': 'master', + 'version': '20171011093314' + }, + { + 'id': 1099, + 'koji_tag': 'module-72e94da1453758d8', + 'name': 'testmodule', + 'owner': 'jkaluza', + 'state_name': 'ready', + 'stream': 'master', + "version": "20171011092951" + } + ], + 'meta': { + 'next': ('http://mbs.fedoraproject.org/module-build-service/1/' + 'module-builds/?state=5&verbose=true&per_page=2&' + 'order_desc_by=id&page=2') + } + } + json_three = { + 'items': [ + { + 'id': 1109, + 'koji_tag': 'module-057fc15e0e44b333', + 'name': 'testmodule', + 'owner': 'mprahl', + 'state_name': 'failed', + 'stream': 'master', + 'version': '20171011173928' + }, + { + 'id': 1094, + 'koji_tag': 'module-640521aea601c6b2', + 'name': 'testmodule', + 'owner': 'mprahl', + 'state_name': 'failed', + 'stream': 'master', + 'version': '20171010151103' + } + ], + 'meta': { + 'next': ('http://mbs.fedoraproject.org/module-build-service/1' + '/module-builds/?state=4&verbose=true&per_page=2&' + 'order_desc_by=id&page=2') + } + } + + mock_rv = Mock() + mock_rv.ok = True + mock_rv.json.side_effect = [json_one, json_two, json_three] + mock_get.return_value = mock_rv + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_overview() + + # Can't confirm the call parameters because multithreading makes the + # order random + self.assertEqual(mock_get.call_count, 3) + output = sys.stdout.getvalue().strip() + expected_output = """ +ID: 1100 +Name: testmodule +Stream: master +Version: 20171011093314 +Koji Tag: module-c24f55c24c8fede1 +Owner: jkaluza +State: ready + +ID: 1109 +Name: testmodule +Stream: master +Version: 20171011173928 +Koji Tag: module-057fc15e0e44b333 +Owner: mprahl +State: failed +""".strip() + self.assertEqual(output, expected_output) + + @patch.object(Commands, '_run_command') + def test_module_build_local(self, mock_run): + """ + Test submitting a local module build + """ + cli_cmd = [ + 'rpkg', + '--path', + self.cloned_repo_path, + 'module-build-local', + 'git://pkgs.fedoraproject.org/modules/testmodule?#79d87a5a', + 'master' + ] + mock_proc = Mock() + mock_proc.returncode = 0 + mock_run.return_value = mock_proc + with patch('sys.argv', new=cli_cmd): + cli = self.new_cli() + cli.module_build_local() + mock_run.assert_called_once_with([ + 'mbs-manager', + 'build_module_locally', + 'git://pkgs.fedoraproject.org/modules/testmodule?#79d87a5a', + 'master'])