From 18391892efaf96eb7f9adceeec2df1d0f3ecf0d6 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Sep 29 2016 13:35:58 +0000 Subject: New source code layout * src/ directory is removed and package pyrpkg is moved to the top of project directory. * Bash completion and configuration file are in dedicated directory. * Script to generate manpage are moved to docs/. Signed-off-by: Chenxiong Qi --- diff --git a/.gitignore b/.gitignore index 630fa85..ff993e6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,6 @@ py-compile /*.tar.gz /*.rpm dist/ -src/rpkg.egg-info/ +rpkg.egg-info/ /sources .coverage diff --git a/MANIFEST.in b/MANIFEST.in index e61caa6..49c0a3c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ include COPYING include COPYING-koji include LGPL -include src/rpkg.bash -include src/rpkg.conf -include src/rpkg_man_page.py +include doc/rpkg_man_page.py +include bin/rpkg +recursive-include tests * +recursive-include etc * diff --git a/bin/rpkg b/bin/rpkg new file mode 100755 index 0000000..6cd10e2 --- /dev/null +++ b/bin/rpkg @@ -0,0 +1,68 @@ +#!/usr/bin/python +# rpkg - a script to interact with the Red Hat Packaging system +# +# Copyright (C) 2011 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +import pyrpkg +import pyrpkg.cli +import pyrpkg.utils +import os +import sys +import logging +from six.moves import configparser +import argparse + +# Setup an argparser and parse the known commands to get the config file +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument('-C', '--config', help='Specify a config file to use', + default='/etc/rpkg/rpkg.conf') + +(args, other) = parser.parse_known_args() + +# Make sure we have a sane config file +if not os.path.exists(args.config) and not other[-1] in ['--help', '-h']: + sys.stderr.write('Invalid config file %s\n' % args.config) + sys.exit(1) + +# Setup a configuration object and read config file data +config = configparser.SafeConfigParser() +config.read(args.config) + +client = pyrpkg.cli.cliClient(config) +client.do_imports() +client.parse_cmdline() + +if not client.args.path: + try: + client.args.path = pyrpkg.utils.getcwd() + except: + print('Could not get current path, have you deleted it?') + sys.exit(1) + +# setup the logger -- This logger will take things of INFO or DEBUG and +# log it to stdout. Anything above that (WARN, ERROR, CRITICAL) will go +# to stderr. Normal operation will show anything INFO and above. +# Quiet hides INFO, while Verbose exposes DEBUG. In all cases WARN or +# higher are exposed (via stderr). +log = pyrpkg.log +client.setupLogging(log) + +if client.args.v: + log.setLevel(logging.DEBUG) +elif client.args.q: + log.setLevel(logging.WARNING) +else: + log.setLevel(logging.INFO) + +# Run the necessary command +try: + sys.exit(client.args.command()) +except KeyboardInterrupt: + pass diff --git a/doc/rpkg_man_page.py b/doc/rpkg_man_page.py new file mode 100644 index 0000000..dcc5c8f --- /dev/null +++ b/doc/rpkg_man_page.py @@ -0,0 +1,158 @@ +# Print a man page from the help texts. +# +# Copyright (C) 2011 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +import sys +import datetime + + +# We could substitute the "" in .TH with the rpkg version if we knew it +man_header = """\ +.\\" man page for rpkg +.TH rpkg 1 "%(today)s" "" "rpm\-packager" +.SH "NAME" +rpkg \- RPM Packaging utility +.SH "SYNOPSIS" +.B "rpkg" +[ +.I global_options +] +.I "command" +[ +.I command_options +] +[ +.I command_arguments +] +.br +.B "rpkg" +.B "help" +.br +.B "rpkg" +.I "command" +.B "\-\-help" +.SH "DESCRIPTION" +.B "rpkg" +is a script to interact with the RPM Packaging system. +""" + +man_footer = """\ +.SH "SEE ALSO" +.UR "https://fedorahosted.org/rpkg/" +.BR "https://fedorahosted.org/rpkg/" +""" + + +class ManFormatter(object): + + def __init__(self, man): + self.man = man + + def write(self, data): + for line in data.split('\n'): + self.man.write(' %s\n' % line) + + +def strip_usage(s): + """Strip "usage: " string from beginning of string if present""" + if s.startswith('usage: '): + return s.replace('usage: ', '', 1) + else: + return s + + +def man_constants(): + """Global constants for man file templates""" + today = datetime.date.today() + today_manstr = today.strftime('%Y\-%m\-%d') + return {'today': today_manstr} + + +def generate(parser, subparsers): + """\ + Generate the man page on stdout + + Given the argparse based parser and subparsers arguments, generate + the corresponding man page and write it to stdout. + """ + + # Not nice, but works: Redirect any print statement output to + # stderr to avoid clobbering the man page output on stdout. + man_file = sys.stdout + sys.stdout = sys.stderr + + mf = ManFormatter(man_file) + + choices = subparsers.choices + k = sorted(choices.keys()) + + man_file.write(man_header % man_constants()) + + helptext = parser.format_help() + helptext = strip_usage(helptext) + helptextsplit = helptext.split('\n') + helptextsplit = [line for line in helptextsplit + if not line.startswith(' -h, --help')] + + man_file.write('.SS "%s"\n' % ("Global Options",)) + + outflag = False + for line in helptextsplit: + if line == "optional arguments:": + outflag = True + elif line == "": + outflag = False + elif outflag: + man_file.write("%s\n" % line) + + help_texts = {} + for pa in subparsers._choices_actions: + help_texts[pa.dest] = getattr(pa, 'help', None) + + man_file.write('.SH "COMMAND OVERVIEW"\n') + + for command in k: + cmdparser = choices[command] + if not cmdparser.add_help: + continue + usage = cmdparser.format_usage() + usage = strip_usage(usage) + usage = ''.join(usage.split('\n')) + usage = ' '.join(usage.split()) + if help_texts[command]: + man_file.write('.TP\n.B "%s"\n%s\n' % (usage, help_texts[command])) + else: + man_file.write('.TP\n.B "%s"\n' % (usage)) + + man_file.write('.SH "COMMAND REFERENCE"\n') + for command in k: + cmdparser = choices[command] + if not cmdparser.add_help: + continue + + man_file.write('.SS "%s"\n' % cmdparser.prog) + + help = help_texts[command] + if help and not cmdparser.description: + if not help.endswith('.'): + help = "%s." % help + cmdparser.description = help + + h = cmdparser.format_help() + mf.write(h) + + man_file.write(man_footer) + + +if __name__ == '__main__': + import pyrpkg.cli + client = pyrpkg.cli.cliClient(name='rpkg', config=None) + generate(client.parser, client.subparsers) diff --git a/etc/bash_completion.d/rpkg.bash b/etc/bash_completion.d/rpkg.bash new file mode 100644 index 0000000..3dce4f7 --- /dev/null +++ b/etc/bash_completion.d/rpkg.bash @@ -0,0 +1,321 @@ +# rpkg bash completion + +_rpkg() +{ + COMPREPLY=() + + in_array() + { + local i + for i in $2; do + [[ $i = $1 ]] && return 0 + done + return 1 + } + + _filedir_exclude_paths() + { + _filedir "$@" + for ((i=0; i<=${#COMPREPLY[@]}; i++)); do + [[ ${COMPREPLY[$i]} =~ /?\.git/? ]] && unset COMPREPLY[$i] + done + } + + local cur prev + # _get_comp_words_by_ref is in bash-completion >= 1.2, which EL-5 lacks. + if type _get_comp_words_by_ref &>/dev/null; then + _get_comp_words_by_ref cur prev + else + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + fi + + # global options + + local options="--help -v -q" + local options_value="--dist --user --path" + local commands="build chain-build ci clean clog clone co container-build container-build-config commit compile copr-build diff gimmespec giturl help \ + gitbuildhash import install lint local mockbuild mock-config new new-sources patch prep pull push scratch-build sources \ + srpm switch-branch tag unused-patches upload verify-files verrel" + + # parse main options and get command + + local command= + local command_first= + local path= + + local i w + for (( i = 0; i < ${#COMP_WORDS[*]} - 1; i++ )); do + w="${COMP_WORDS[$i]}" + # option + if [[ ${w:0:1} = - ]]; then + if in_array "$w" "$options_value"; then + ((i++)) + [[ "$w" = --path ]] && path="${COMP_WORDS[$i]}" + fi + # command + elif in_array "$w" "$commands"; then + command="$w" + command_first=$((i+1)) + break + fi + done + + # complete base options + + if [[ -z $command ]]; then + if [[ $cur == -* ]]; then + COMPREPLY=( $(compgen -W "$options $options_value" -- "$cur") ) + return 0 + fi + + case "$prev" in + --config) + _filedir_exclude_paths + ;; + --dist) + ;; + --user|-u) + ;; + --path) + _filedir_exclude_paths + ;; + *) + COMPREPLY=( $(compgen -W "$commands" -- "$cur") ) + ;; + esac + + return 0 + fi + + # parse command specific options + + local options= + local options_target= options_arches= options_branch= options_string= options_file= options_dir= options_srpm= + local after= after_more= + + case $command in + help|gimmespec|gitbuildhash|giturl|lint|new|unused-patches|verrel) + ;; + build) + options="--nowait --background --skip-tag --scratch --md5" + options_arches="--arches" + options_srpm="--srpm" + options_target="--target" + ;; + chain-build) + options="--nowait --background" + options_target="--target" + after="package" + after_more=true + ;; + clean) + options="--dry-run -x" + ;; + clog) + options="--raw" + ;; + clone|co) + options="--branches --anonymous" + options_branch="-b" + after="package" + ;; + container-build) + options="--scratch --nowait" + options_target="--target" + options_string="--repo-url" + ;; + container-build-config) + options="--get-autorebuild" + options_bool="--set-autorebuild" + ;; + commit|ci) + options="--push --clog --raw --tag" + options_string="--message" + options_file="--file" + after="file" + after_more=true + ;; + compile|install) + options="--short-circuit --nocheck" + options_arch="--arch" + options_dir="--builddir" + ;; + copr-build) + options="--nowait" + after="package" + after_more=true + ;; + diff) + options="--cached" + after="file" + after_more=true + ;; + import) + options="--create" + options_branch="--branch" + after="srpm" + ;; + lint) + options="--info" + options_file="--rpmlintconf" + ;; + local) + options="--md5" + options_arch="--arch" + options_dir="--builddir" + ;; + mock-config) + options="--target" + options_arch="--arch" + ;; + mockbuild) + options="--md5 --no-clean --no-cleanup-after --no-clean-all" + options_mroot="--root" + ;; + patch) + options="--rediff" + options_string="--suffix" + ;; + prep|verify-files) + options_arch="--arch" + options_dir="--builddir" + ;; + pull) + options="--rebase --no-rebase" + ;; + push) + options="--force" + ;; + scratch-build) + options="--nowait --background --md5" + options_target="--target" + options_arches="--arches" + options_srpm="--srpm" + ;; + sources) + options_dir="--outdir" + ;; + srpm) + options="--md5" + ;; + switch-branch) + options="--list" + after="branch" + ;; + tag) + options="--clog --raw --force --list --delete" + options_string="--message" + options_file="--file" + after_more=true + ;; + upload|new-sources) + after="file" + after_more=true + ;; + esac + + local all_options="--help $options" + local all_options_value="$options_target $options_arches $options_branch $options_string $options_file $options_dir $options_srpm $options_bool" + + # count non-option parameters + + local i w + local last_option= + local after_counter=0 + for (( i = $command_first; i < ${#COMP_WORDS[*]} - 1; i++)); do + w="${COMP_WORDS[$i]}" + if [[ ${w:0:1} = - ]]; then + if in_array "$w" "$all_options"; then + last_option="$w" + continue + elif in_array "$w" "$all_options_value"; then + last_option="$w" + ((i++)) + continue + fi + fi + in_array "$last_option" "$options_arches" || ((after_counter++)) + done + + # completion + + if [[ -n $options_target ]] && in_array "$prev" "$options_target"; then + COMPREPLY=( $(compgen -W "$(_rpkg_target)" -- "$cur") ) + + elif [[ -n $options_arches ]] && in_array "$last_option" "$options_arches"; then + COMPREPLY=( $(compgen -W "$(_rpkg_arch) $all_options" -- "$cur") ) + + elif [[ -n $options_srpm ]] && in_array "$prev" "$options_srpm"; then + _filedir_exclude_paths "*.src.rpm" + + elif [[ -n $options_branch ]] && in_array "$prev" "$options_branch"; then + COMPREPLY=( $(compgen -W "$(_rpkg_branch "$path")" -- "$cur") ) + + elif [[ -n $options_file ]] && in_array "$prev" "$options_file"; then + _filedir_exclude_paths + + elif [[ -n $options_dir ]] && in_array "$prev" "$options_dir"; then + _filedir_exclude_paths -d + + elif [[ -n $options_bool ]] && in_array "$prev" "$options_bool"; then + COMPREPLY=( $(compgen -W "true false" -- "$cur") ) + + elif [[ -n $options_string ]] && in_array "$prev" "$options_string"; then + COMPREPLY=( ) + + else + local after_options= + + if [[ $after_counter -eq 0 ]] || [[ $after_more = true ]]; then + case $after in + file) _filedir_exclude_paths ;; + srpm) _filedir_exclude_paths "*.src.rpm" ;; + branch) after_options="$(_rpkg_branch "$path")" ;; + package) after_options="$(_rpkg_package "$cur")";; + esac + fi + + if [[ $cur != -* ]]; then + all_options= + all_options_value= + fi + + COMPREPLY+=( $(compgen -W "$all_options $all_options_value $after_options" -- "$cur" ) ) + fi + + return 0 +} && +complete -F _rpkg rpkg + +_rpkg_target() +{ + koji list-targets --quiet 2>/dev/null | cut -d" " -f1 +} + +_rpkg_arch() +{ + echo "i386 x86_64 ppc ppc64 s390 s390x sparc sparc64" +} + +_rpkg_branch() +{ + local git_options= format="--format %(refname:short)" + [[ -n $1 ]] && git_options="--git-dir=$1/.git" + + git $git_options for-each-ref $format 'refs/remotes' | sed 's,.*/,,' + git $git_options for-each-ref $format 'refs/heads' +} + +_rpkg_package() +{ + repoquery -C --qf=%{sourcerpm} "$1*" 2>/dev/null | sort -u | sed -r 's/(-[^-]*){2}\.src\.rpm$//' +} + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/etc/rpkg/rpkg.conf b/etc/rpkg/rpkg.conf new file mode 100644 index 0000000..40b6f15 --- /dev/null +++ b/etc/rpkg/rpkg.conf @@ -0,0 +1,11 @@ +[rpkg] +lookaside = http://localhost/repo/pkgs +lookasidehash = md5 +lookaside_cgi = https://localhost/repo/pkgs/upload.cgi +gitbaseurl = ssh://%(user)s@localhost/%(module)s +anongiturl = git://localhost/%(module)s +branchre = f\d$|f\d\d$|el\d$|olpc\d$|master$ +kojiconfig = /etc/koji.conf +build_client = koji +clone_config = + bz.default-component %(module)s diff --git a/pyrpkg/__init__.py b/pyrpkg/__init__.py new file mode 100644 index 0000000..17abde0 --- /dev/null +++ b/pyrpkg/__init__.py @@ -0,0 +1,2590 @@ +# pyrpkg - a Python library for RPM Packagers +# +# Copyright (C) 2011 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + +import errno +import fnmatch +import git +import glob +import koji +import logging +import os +import posixpath +import pwd +import re +import rpm +import shutil +import six +import sys +import tempfile + +from ConfigParser import ConfigParser + +from osbs.api import OSBS +from osbs.conf import Configuration +from six.moves import configparser +from six.moves import urllib + +from pyrpkg.errors import HashtypeMixingError, rpkgError, rpkgAuthError, \ + UnknownTargetError +from .gitignore import GitIgnore +from pyrpkg.lookaside import CGILookasideCache +from pyrpkg.sources import SourcesFile +from pyrpkg.utils import cached_property, log_result + +if sys.version_info[0:2] >= (2, 5): + import subprocess +else: + # We need a subprocess that has check_call + from kitchen.pycompat27 import subprocess + +# Try to import krb, it's OK if it fails +try: + import krbV +except ImportError: + pass + + +class NullHandler(logging.Handler): + """Null logger to avoid spurious messages, add a handler in app code""" + def emit(self, record): + pass + + +h = NullHandler() +# This is our log object, clients of this library can use this object to +# define their own logging needs +log = logging.getLogger(__name__) +# Add the null handler +log.addHandler(h) + + +class Commands(object): + """This is a class to hold all the commands that will be called + by clients + """ + + # This shouldn't change... often + UPLOADEXTS = ['tar', 'gz', 'bz2', 'lzma', 'xz', 'Z', 'zip', 'tff', + 'bin', 'tbz', 'tbz2', 'tgz', 'tlz', 'txz', 'pdf', 'rpm', + 'jar', 'war', 'db', 'cpio', 'jisp', 'egg', 'gem', 'spkg', + 'oxt', 'xpi'] + + def __init__(self, path, lookaside, lookasidehash, lookaside_cgi, + gitbaseurl, anongiturl, branchre, kojiconfig, + build_client, user=None, + dist=None, target=None, quiet=False, + distgit_namespaced=False): + """Init the object and some configuration details.""" + + # Path to operate on, most often pwd + self._path = None + self.path = os.path.abspath(path) + # The url of the lookaside for source archives + self.lookaside = lookaside + # The type of hash to use with the lookaside + self.lookasidehash = lookasidehash + # The CGI server for the lookaside + self.lookaside_cgi = lookaside_cgi + # The base URL of the git server + self.gitbaseurl = gitbaseurl + # The anonymous version of the git url + self.anongiturl = anongiturl + # The regex of branches we care about + self.branchre = branchre + # The location of the buildsys config file + self.kojiconfig = os.path.expanduser(kojiconfig) + # The buildsys client to use + self.build_client = build_client + # A way to override the discovered "distribution" + self.dist = dist + # Set the default hashtype + self.hashtype = 'sha256' + # Set an attribute for quiet or not + self.quiet = quiet + # Set place holders for properties + # Anonymous buildsys session + self._anon_kojisession = None + # The upstream branch a downstream branch is tracking + self._branch_merge = None + # The latest commit + self._commit = None + # The disttag rpm value + self._disttag = None + # The distval rpm value + self._distval = None + # The distvar rpm value + self._distvar = None + # The rpm epoch of the cloned module + self._epoch = None + # An authenticated buildsys session + self._kojisession = None + # A web url of the buildsys server + self._kojiweburl = None + # The local arch to use in rpm building + self._localarch = None + # A property to load the mock config + self._mockconfig = None + # The name of the cloned module + self._module_name = None + # The distgit namespaced name of the cloned module + self._ns_module_name = None + # The name of the module from spec file + self._module_name_spec = None + # The rpm name-version-release of the cloned module + self._nvr = None + # The rpm release of the cloned module + self._rel = None + # The cloned repo object + self._repo = None + # The rpm defines used when calling rpm + self._rpmdefines = None + # The specfile in the cloned module + self._spec = None + # The build target within the buildsystem + self._target = target + # The top url to our build server + self._topurl = None + # The user to use or discover + self._user = user + # The password to use + self._password = None + # The alternate Koji user to run commands as + self._runas = None + # The rpm version of the cloned module + self._ver = None + self.log = log + # Pushurl or url of remote of branch + self._push_url = None + # Name of remote determined from current clone + self._branch_remote = None + # Name of default remote to be used for new clone + self.default_branch_remote = 'origin' + # Default sources file output format type + self.source_entry_type = 'old' + # Set an attribute debug + self.debug = False + # Set an attribute verbose + self.verbose = False + # Config to set after cloning + self.clone_config = None + # Git namespacing for more than just rpm build artifacts + self.distgit_namespaced = distgit_namespaced + + # Define properties here + # Properties allow us to "lazy load" various attributes, which also means + # that we can do clone actions without knowing things like the spec + # file or rpm data. + + @cached_property + def lookasidecache(self): + """A helper to interact with the lookaside cache + + This is a pyrpkg.lookaside.CGILookasideCache instance, providing all + the needed stuff to communicate with a Fedora-style lookaside cache. + + Downstream users of the pyrpkg API may override this property with + their own, returning their own implementation of a lookaside cache + helper object. + """ + return CGILookasideCache( + self.lookasidehash, self.lookaside, self.lookaside_cgi, + client_cert=self.cert_file, ca_cert=self.ca_cert) + + @property + def path(self): + return self._path + + @path.setter + def path(self, value): + if self._path != value: + # Ensure all properties which depend on self.path will be + # freshly loaded next time + self._push_url = None + self._branch_remote = None + self._repo = None + self._ns_module_name = None + self._path = value + + @property + def anon_kojisession(self): + """This property ensures the anon kojisession attribute""" + + if not self._anon_kojisession: + self.load_kojisession(anon=True) + return self._anon_kojisession + + def load_kojisession(self, anon=False): + """Initiate a koji session. + + The koji session can be logged in or anonymous + """ + + # Stealing a bunch of code from /usr/bin/koji here, too bad it isn't + # in a more usable library form + defaults = { + 'server': None, + 'topurl': 'http://localhost/kojiroot', + 'weburl': 'http://localhost/koji', + 'cert': '~/.koji/client.crt', + 'ca': '~/.koji/clientca.crt', + 'serverca': '~/.koji/serverca.crt', + 'authtype': None, + 'krbservice': None, + 'timeout': None, + 'keepalive': True, + 'max_retries': None, + 'retry_interval': None, + 'anon_retry': True, + 'offline_retry': None, + 'offline_retry_interval': None, + 'use_fast_upload': None, + 'debug': None, + 'debug_xmlrpc': None + } + + # Process the configs in order, global, user, then any option passed + config = configparser.ConfigParser() + confs = [self.kojiconfig, + os.path.expanduser('~/.koji/config')] + config.read(confs) + + if config.has_section(os.path.basename(self.build_client)): + for name, value in config.items(os.path.basename( + self.build_client)): + if name in defaults: + if name in ('keepalive', 'anon_retry', 'offline_retry', + 'use_fast_upload', + 'debug', 'debug_xmlrpc'): + defaults[name] = config.getboolean( + os.path.basename(self.build_client), name) + elif name in ('timeout', 'max_retries', 'retry_interval', + 'offline_retry_interval'): + defaults[name] = config.getint( + os.path.basename(self.build_client), name) + else: + defaults[name] = value + if not defaults['server']: + raise rpkgError('No server defined in: %s' % ', '.join(confs)) + # Expand out the directory options + for name in ('cert', 'ca', 'serverca'): + if defaults[name]: + defaults[name] = os.path.expanduser(defaults[name]) + self.log.debug('Initiating a %s session to %s', + os.path.basename(self.build_client), defaults['server']) + session_opts = {} + for name in ('krbservice', 'timeout', 'keepalive', + 'max_retries', 'retry_interval', 'anon_retry', + 'offline_retry', 'offline_retry_interval', + 'debug', 'debug_xmlrpc', + 'use_fast_upload'): + if defaults[name] is not None: + session_opts[name] = defaults[name] + try: + if anon: + self._anon_kojisession = koji.ClientSession(defaults['server'], + session_opts) + else: + self._kojisession = koji.ClientSession(defaults['server'], + session_opts) + except: + raise rpkgError('Could not initiate %s session' % + os.path.basename(self.build_client)) + # save the weburl and topurl for later use as well + self._kojiweburl = defaults['weburl'] + self._topurl = defaults['topurl'] + if not anon: + # Default to ssl if not otherwise specified and we have the cert + if defaults['authtype'] == 'ssl' or \ + os.path.isfile(defaults['cert']) and \ + defaults['authtype'] is None: + try: + self._kojisession.ssl_login(defaults['cert'], + defaults['ca'], + defaults['serverca'], + proxyuser=self.runas) + except koji.ssl.SSLCommon.SSL.Error as error: + for (_, _, ssl_reason) in error.message: + # Use heuristic. Some OpenSSL libs doesn't store error + # codes + if 'certificate revoked' in ssl_reason or \ + 'certificate expired' in ssl_reason: + self.log.info("Certificate is revoked or expired.") + raise rpkgAuthError('Could not auth with koji. Login ' + 'failed: %s' % error) + # Or try password auth + elif defaults['authtype'] == 'password' or self.password \ + and defaults['authtype'] is None: + if self.runas: + raise rpkgError('--runas cannot be used with password auth') + self._kojisession.opts['user'] = self.user + self._kojisession.opts['password'] = self.password + self._kojisession.login() + # Or try kerberos + elif defaults['authtype'] == 'kerberos' or self._has_krb_creds() \ + and defaults['authtype'] is None: + self._kojisession.krb_login(proxyuser=self.runas) + if not self._kojisession.logged_in: + raise rpkgError('Could not login to %s' % defaults['server']) + + @property + def branch_merge(self): + """This property ensures the branch attribute""" + + if not self._branch_merge: + self.load_branch_merge() + return(self._branch_merge) + + def load_branch_merge(self): + """Find the remote tracking branch from the branch we're on. + + The goal of this function is to catch if we are on a branch we + + can make some assumptions about. If there is no merge point + + then we raise and ask the user to specify. + """ + + if self.dist: + self._branch_merge = self.dist + else: + try: + localbranch = self.repo.active_branch.name + except TypeError as e: + raise rpkgError('Repo in inconsistent state: %s' % e) + try: + merge = self.repo.git.config('--get', + 'branch.%s.merge' % localbranch) + except git.GitCommandError as e: + raise rpkgError('Unable to find remote branch. Use --dist') + # Trim off the refs/heads so that we're just working with + # the branch name + merge = merge.replace('refs/heads/', '') + self._branch_merge = merge + + @property + def branch_remote(self): + """This property ensures the branch_remote attribute""" + + if not self._branch_remote: + self.load_branch_remote() + return self._branch_remote + + def load_branch_remote(self): + """Find the name of remote from branch we're on.""" + + try: + remote = self.repo.git.config('--get', 'branch.%s.remote' + % self.branch_merge) + except (git.GitCommandError, rpkgError) as e: + remote = self.default_branch_remote + self.log.debug("Could not determine the remote name: %s", str(e)) + self.log.debug("Falling back to default remote name '%s'", remote) + + self._branch_remote = remote + + @property + def push_url(self): + """This property ensures the push_url attribute""" + + if not self._push_url: + self.load_push_url() + return self._push_url + + def load_push_url(self): + """Find the pushurl or url of remote of branch we're on.""" + try: + url = self.repo.git.remote('get-url', '--push', self.branch_remote) + except git.GitCommandError as e: + try: + url = self.repo.git.config( + '--get', 'remote.%s.pushurl' % self.branch_remote) + except git.GitCommandError as e: + try: + url = self.repo.git.config( + '--get', 'remote.%s.url' % self.branch_remote) + except git.GitCommandError as e: + raise rpkgError('Unable to find remote push url: %s' % e) + if isinstance(url, six.text_type): + # GitPython >= 1.0 return unicode. It must be encoded to string. + self._push_url = url.encode('utf-8') + else: + self._push_url = url + + @property + def commithash(self): + """This property ensures the commit attribute""" + + if not self._commit: + self.load_commit() + return self._commit + + def load_commit(self): + """Discover the latest commit to the package""" + + # Get the commit hash + comobj = six.next(self.repo.iter_commits()) + # Work around different versions of GitPython + if hasattr(comobj, 'sha'): + self._commit = comobj.sha + else: + self._commit = comobj.hexsha + + @property + def disttag(self): + """This property ensures the disttag attribute""" + + if not self._disttag: + self.load_rpmdefines() + return self._disttag + + @property + def distval(self): + """This property ensures the distval attribute""" + + if not self._distval: + self.load_rpmdefines() + return self._distval + + @property + def distvar(self): + """This property ensures the distvar attribute""" + + if not self._distvar: + self.load_rpmdefines() + return self._distvar + + @property + def epoch(self): + """This property ensures the epoch attribute""" + + if not self._epoch: + self.load_nameverrel() + return self._epoch + + @property + def kojisession(self): + """This property ensures the kojisession attribute""" + + if not self._kojisession: + self.load_kojisession() + return self._kojisession + + @property + def kojiweburl(self): + """This property ensures the kojiweburl attribute""" + + if not self._kojiweburl: + self.load_kojisession() + return self._kojiweburl + + @property + def localarch(self): + """This property ensures the module attribute""" + + if not self._localarch: + self.load_localarch() + return(self._localarch) + + def load_localarch(self): + """Get the local arch as defined by rpm""" + + proc = subprocess.Popen(['rpm --eval %{_arch}'], shell=True, + stdout=subprocess.PIPE) + self._localarch = proc.communicate()[0].strip('\n') + + @property + def mockconfig(self): + """This property ensures the mockconfig attribute""" + + if not self._mockconfig: + self.load_mockconfig() + return self._mockconfig + + @mockconfig.setter + def mockconfig(self, config): + self._mockconfig = config + + def load_mockconfig(self): + """This sets the mockconfig attribute""" + + self._mockconfig = '%s-%s' % (self.target, self.localarch) + + @property + def module_name(self): + """This property ensures the module attribute""" + + if not self._module_name: + self.load_module_name() + return self._module_name + + @module_name.setter + def module_name(self, module_name): + self._module_name = module_name + + def load_module_name(self): + """Loads a package module.""" + + try: + if self.push_url: + parts = urllib.parse.urlparse(self.push_url) + + # FIXME + # if self.distgit_namespaced: + # self._module_name = "/".join(parts.path.split("/")[-2:]) + module_name = posixpath.basename(parts.path) + + if module_name.endswith(b'.git'): + module_name = module_name[:-len(b'.git')] + self._module_name = module_name + return + except rpkgError: + self.log.warning('Failed to get module name from Git url or pushurl') + + self.load_nameverrel() + if self._module_name_spec: + self._module_name = self._module_name_spec + return + + raise rpkgError('Could not find current module name.' + ' Use --module-name.') + + @property + def ns_module_name(self): + """This property ensures the module attribute""" + + if not self._ns_module_name: + self.load_ns_module_name() + return self._ns_module_name + + @ns_module_name.setter + def ns_module_name(self, ns_module_name): + self._ns_module_name = ns_module_name + + def _print_old_checkout_warning(self, module): + self.log.warning('Your git configuration does not use a namespace.') + self.log.warning('Consider updating your git configuration by running:') + self.log.warning(' git remote set-url %s %s', + self.branch_remote, self._get_namespace_giturl(module)) + + def load_ns_module_name(self): + """Loads a package module.""" + + try: + if self.push_url: + parts = urllib.parse.urlparse(self.push_url) + + if self.distgit_namespaced: + path_parts = [p for p in parts.path.split("/") if p] + if len(path_parts) == 1: + self._print_old_checkout_warning(path_parts[0]) + path_parts.insert(0, "rpms") + ns_module_name = "/".join(path_parts[-2:]) + else: + ns_module_name = posixpath.basename(parts.path) + + if ns_module_name.endswith('.git'): + ns_module_name = ns_module_name[:-len('.git')] + self._ns_module_name = ns_module_name + return + except rpkgError: + self.log.warning('Failed to get ns_module_name from Git url or pushurl') + + @property + def nvr(self): + """This property ensures the nvr attribute""" + + if not self._nvr: + self.load_nvr() + return self._nvr + + def load_nvr(self): + """This sets the nvr attribute""" + + self._nvr = '%s-%s-%s' % (self.module_name, self.ver, self.rel) + + @property + def rel(self): + """This property ensures the rel attribute""" + if not self._rel: + self.load_nameverrel() + return(self._rel) + + def load_nameverrel(self): + """Set the release of a package module.""" + + cmd = ['rpm'] + cmd.extend(self.rpmdefines) + # We make sure there is a space at the end of our query so that + # we can split it later. When there are subpackages, we get a + # listing for each subpackage. We only care about the first. + cmd.extend(['-q', '--qf', '"%{NAME} %{EPOCH} %{VERSION} %{RELEASE}??"', + '--specfile', '"%s"' % os.path.join(self.path, self.spec)]) + joined_cmd = ' '.join(cmd) + try: + proc = subprocess.Popen(joined_cmd, shell=True, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, err = proc.communicate() + except Exception as e: + if err: + self.log.debug('Errors occoured while running following command to get N-V-R-E:') + self.log.debug(joined_cmd) + self.log.error(err) + raise rpkgError('Could not query n-v-r of %s: %s' + % (self.module_name, e)) + if err: + self.log.debug('Errors occoured while running following command to get N-V-R-E:') + self.log.debug(joined_cmd) + self.log.error(err) + # Get just the output, then split it by ??, grab the first and split + # again to get ver and rel + first_line_output = output.split('??')[0] + parts = first_line_output.split() + if len(parts) != 4: + raise rpkgError('Could not get n-v-r-e from %r' + % first_line_output) + (self._module_name_spec, + self._epoch, + self._ver, + self._rel) = parts + + # Most packages don't include a "Epoch: 0" line, in which case RPM + # returns '(none)' + if self._epoch == "(none)": + self._epoch = "0" + + @property + def repo(self): + """This property ensures the repo attribute""" + + if not self._repo: + self.load_repo() + return(self._repo) + + def load_repo(self): + """Create a repo object from our path""" + + self.log.debug('Creating repo object from %s', self.path) + try: + self._repo = git.Repo(self.path) + except git.InvalidGitRepositoryError: + raise rpkgError('%s is not a valid repo' % self.path) + + @property + def rpmdefines(self): + """This property ensures the rpm defines""" + + if not self._rpmdefines: + self.load_rpmdefines() + return(self._rpmdefines) + + def load_rpmdefines(self): + """Populate rpmdefines based on current active branch""" + + # This is another function ripe for subclassing + + try: + # This regex should find the 'rhel-5' or 'rhel-6.2' parts of the + # branch name. There should only be one of those, and all branches + # should end in one. + osver = re.search(r'rhel-\d.*$', self.branch_merge).group() + except AttributeError: + raise rpkgError('Could not find the base OS ver from branch name' + ' %s. Consider using --dist option' % + self.branch_merge) + self._distvar, self._distval = osver.split('-') + self._distval = self._distval.replace('.', '_') + self._disttag = 'el%s' % self._distval + self._rpmdefines = ["--define '_sourcedir %s'" % self.path, + "--define '_specdir %s'" % self.path, + "--define '_builddir %s'" % self.path, + "--define '_srcrpmdir %s'" % self.path, + "--define '_rpmdir %s'" % self.path, + "--define 'dist .%s'" % self._disttag, + "--define '%s %s'" % (self._distvar, + self._distval.split('_')[0]), + # int and float this to remove the decimal + "--define '%s 1'" % self._disttag] + + @property + def spec(self): + """This property ensures the module attribute""" + + if not self._spec: + self.load_spec() + return self._spec + + def load_spec(self): + """This sets the spec attribute""" + + deadpackage = False + + # Get a list of files in the path we're looking at + files = os.listdir(self.path) + # Search the files for the first one that ends with ".spec" + for f in files: + if f.endswith('.spec') and not f.startswith('.'): + self._spec = f + return + if f == 'dead.package': + deadpackage = True + if deadpackage: + raise rpkgError('No spec file found. This package is retired') + else: + raise rpkgError('No spec file found.') + + @property + def target(self): + """This property ensures the target attribute""" + + if not self._target: + self.load_target() + return self._target + + def load_target(self): + """This creates the target attribute based on branch merge""" + + # If a site has a different naming scheme, this would be where + # a site would override + self._target = '%s-candidate' % self.branch_merge + + @property + def topurl(self): + """This property ensures the topurl attribute""" + + if not self._topurl: + # Assume anon here, whatever. + self.load_kojisession(anon=True) + return self._topurl + + @property + def user(self): + """This property ensures the user attribute""" + + if not self._user: + self.load_user() + return self._user + + def load_user(self): + """This sets the user attribute""" + + # If a site figures out the user differently (like from ssl cert) + # this is where you'd override and make that happen + self._user = pwd.getpwuid(os.getuid())[0] + + @property + def password(self): + """This property ensures the password attribute""" + + return self._password + + @password.setter + def password(self, password): + self._password = password + + @property + def runas(self): + """This property ensures the runas attribute""" + + return self._runas + + @runas.setter + def runas(self, runas): + self._runas = runas + + @property + def ver(self): + """This property ensures the ver attribute""" + if not self._ver: + self.load_nameverrel() + return(self._ver) + + @property + def mock_results_dir(self): + return os.path.join(self.path, "results_%s" % self.module_name, + self.ver, self.rel) + + @property + def sources_filename(self): + return os.path.join(self.path, 'sources') + + @property + def osbs_config_filename(self): + return os.path.join(self.path, '.osbs-repo-config') + + @property + def cert_file(self): + """A client-side certificate for SSL authentication + + Downstream users of the pyrpkg API should override this property if + they actually need to use a client-side certificate. + + This defaults to None, which means no client-side certificate is used. + """ + return None + + @property + def ca_cert(self): + """A CA certificate to authenticate the server in SSL connections + + Downstream users of the pyrpkg API should override this property if + they actually need to use a CA certificate, usually because their + lookaside cache is using HTTPS with a self-signed certificate. + + This defaults to None, which means the system CA bundle is used. + """ + return None + + # Define some helper functions, they start with _ + def _has_krb_creds(self): + # This function is lifted from /usr/bin/koji + if 'krbV' not in sys.modules: + return False + try: + ctx = krbV.default_context() + ccache = ctx.default_ccache() + princ = ccache.principal() # noqa + return True + except krbV.Krb5Error: + return False + + def _run_command(self, cmd, shell=False, env=None, pipe=[], cwd=None): + """Run the given command. + + _run_command is able to run single command or two commands via pipe. + Whatever the way to run the command, output to both stdout and stderr + will not be captured and output to terminal directly, that is useful + for caller to redirect. + + cmd is a list of the command and arguments + + shell is whether to run in a shell or not, defaults to False + + env is a dict of environment variables to use (if any) + + pipe is a command to pipe the output of cmd into + + cwd is the optional directory to run the command from + + Raises on error, or returns nothing. + """ + + # Process any environment variables. + environ = os.environ + if env: + for item in env.keys(): + self.log.debug('Adding %s:%s to the environment', item, env[item]) + environ[item] = env[item] + # Check if we're supposed to be on a shell. If so, the command must + # be a string, and not a list. + command = cmd + pipecmd = pipe + if shell: + command = ' '.join(cmd) + pipecmd = ' '.join(pipe) + + if pipe: + self.log.debug('Running: %s | %s', ' '.join(cmd), ' '.join(pipe)) + else: + self.log.debug('Running: %s', ' '.join(cmd)) + + try: + if pipe: + # We're piping the stderr over as well, which is probably a + # bad thing, but rpmbuild likes to put useful data on + # stderr, so.... + proc = subprocess.Popen(command, env=environ, shell=shell, cwd=cwd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess.check_call(pipecmd, env=environ, shell=shell, cwd=cwd, stdin=proc.stdout) + else: + subprocess.check_call(command, env=environ, shell=shell, cwd=cwd) + except (subprocess.CalledProcessError, OSError) as e: + raise rpkgError(e) + except KeyboardInterrupt: + raise rpkgError('Command is terminated by user.') + except Exception as e: + raise rpkgError(e) + + def _newer(self, file1, file2): + """Compare the last modification time of the given files + + Returns True is file1 is newer than file2 + + """ + + return os.path.getmtime(file1) > os.path.getmtime(file2) + + def _get_build_arches_from_spec(self): + """Given the path to an spec, retrieve the build arches + + """ + + spec = os.path.join(self.path, self.spec) + try: + hdr = rpm.spec(spec) + except Exception: + raise rpkgError('%s is not a spec file' % spec) + archlist = [pkg.header['arch'] for pkg in hdr.packages] + if not archlist: + raise rpkgError('No compatible build arches found in %s' % spec) + return archlist + + def _get_build_arches_from_srpm(self, srpm, arches): + """Given the path to an srpm, determine the possible build arches + + Use supplied arches as a filter, only return compatible arches + + """ + + archlist = arches + hdr = koji.get_rpm_header(srpm) + if hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1: + raise rpkgError('%s is not a source package.' % srpm) + buildarchs = hdr[rpm.RPMTAG_BUILDARCHS] + exclusivearch = hdr[rpm.RPMTAG_EXCLUSIVEARCH] + excludearch = hdr[rpm.RPMTAG_EXCLUDEARCH] + # Reduce by buildarchs + if buildarchs: + archlist = [a for a in archlist if a in buildarchs] + # Reduce by exclusive arches + if exclusivearch: + archlist = [a for a in archlist if a in exclusivearch] + # Reduce by exclude arch + if excludearch: + archlist = [a for a in archlist if a not in excludearch] + # do the noarch thing + if 'noarch' not in excludearch and ('noarch' in buildarchs or + 'noarch' in exclusivearch): + archlist.append('noarch') + # See if we have anything compatible. Should we raise here? + if not archlist: + raise rpkgError('No compatible build arches found in %s' % srpm) + return archlist + + def _guess_hashtype(self): + """Attempt to figure out the hash type based on branch data""" + + # We may not be able to determine the rpmdefine, if so, fall back. + try: + # This works, except for the small range of Fedoras + # between FC5 and FC12 or so. Nobody builds for that old + # anyway. + if int(re.search(r'\d+', self.distval).group()) < 6: + return('md5') + except: + # An error here is OK, don't bother the user. + pass + + # Fall back to the default hash type + return(self.hashtype) + + def _fetch_remotes(self): + self.log.debug('Fetching remotes') + for remote in self.repo.remotes: + self.repo.git.fetch(remote) + + def _list_branches(self, fetch=True): + """Returns a tuple of local and remote branch names""" + + if fetch: + self._fetch_remotes() + self.log.debug('Listing refs') + refs = self.repo.refs + # Sort into local and remote branches + remotes = [] + locals = [] + for ref in refs: + if type(ref) == git.Head: + self.log.debug('Found local branch %s', ref.name) + locals.append(ref.name) + elif type(ref) == git.RemoteReference: + if ref.remote_head == 'HEAD': + self.log.debug('Skipping remote branch alias HEAD') + continue # Not useful in this context + self.log.debug('Found remote branch %s', ref.name) + remotes.append(ref.name) + return (locals, remotes) + + def _srpmdetails(self, srpm): + """Return a tuple of package name, package files, and upload files.""" + + # get the name + cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME}', srpm] + # Run the command + self.log.debug('Running: %s', ' '.join(cmd)) + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + except OSError as e: + raise rpkgError(e) + name = output + if error: + raise rpkgError('Error querying srpm: %s' % error) + + # now get the files and upload files + files = [] + uploadfiles = [] + cmd = ['rpm', '-qpl', srpm] + self.log.debug('Running: %s', ' '.join(cmd)) + env = dict(os.environ) + env["LANG"] = "C" + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + output, error = proc.communicate() + except OSError as e: + raise rpkgError(e) + # work around signed SRPMs, for these rpm -qpl might print a warning + # like: + # warning: foo-0.0.src.rpm Header V3 RSA/SHA256 Signature, key ID + # fd431d51: NOKEY + if error and not error.startswith("warning:") and "NOKEY" not in error: + raise rpkgError('Error querying srpm: %s' % error) + contents = output.strip().split('\n') + # Cycle through the stuff and sort correctly by its extension + for file in contents: + if file.rsplit('.')[-1] in self.UPLOADEXTS: + uploadfiles.append(file) + else: + files.append(file) + + return((name, files, uploadfiles)) + + def _get_namespace_giturl(self, module): + """Get the namespaced git url, if DistGit namespaces enabled + + Takes a module name + + Returns a string of giturl + + """ + + if self.distgit_namespaced: + if '/' in module: + giturl = self.gitbaseurl % \ + {'user': self.user, 'module': module} + else: + # Default to rpms namespace for backwards compat + giturl = self.gitbaseurl % \ + {'user': self.user, 'module': "rpms/%s" % module} + else: + giturl = self.gitbaseurl % \ + {'user': self.user, 'module': module} + + return giturl + + def _get_namespace_anongiturl(self, module): + """Get the namespaced git url, if DistGit namespaces enabled + + Takes a module name + + Returns a string of giturl + + """ + + if self.distgit_namespaced: + if '/' in module: + giturl = self.anongiturl % {'module': module} + else: + # Default to rpms namespace for backwards compat + giturl = self.anongiturl % {'module': "rpms/%s" % module} + else: + giturl = self.anongiturl % {'module': module} + + return giturl + + def add_tag(self, tagname, force=False, message=None, file=None): + """Add a git tag to the repository + + Takes a tagname + + Optionally can force the tag, include a message, + or reference a message file. + + Runs the tag command and returns nothing + + """ + + cmd = ['git', 'tag'] + cmd.extend(['-a']) + # force tag creation, if tag already exists + if force: + cmd.extend(['-f']) + # Description for the tag + if message: + cmd.extend(['-m', message]) + elif file: + cmd.extend(['-F', os.path.abspath(file)]) + cmd.append(tagname) + # make it so + self._run_command(cmd, cwd=self.path) + self.log.info('Tag \'%s\' was created', tagname) + + def clean(self, dry=False, useignore=True): + """Clean a module checkout of untracked files. + + Can optionally perform a dry-run + + Can optionally not use the ignore rules + + Logs output and returns nothing + + """ + + # setup the command, this could probably be done with some python api... + cmd = ['git', 'clean', '-f', '-d'] + if dry: + cmd.append('--dry-run') + if not useignore: + cmd.append('-x') + if self.quiet: + cmd.append('-q') + # Run it! + self._run_command(cmd, cwd=self.path) + return + + def clone(self, module, path=None, branch=None, bare_dir=None, + anon=False, target=None): + """Clone a repo, optionally check out a specific branch. + + module is the name of the module to clone + + path is the basedir to perform the clone in + + branch is the name of a branch to checkout instead of /master + + bare_dir is the name of a directory to make a bare clone to, if this + is a bare clone. None otherwise. + + anon is whether or not to clone anonymously + + target is the name of the folder in which to clone the repo + + Logs the output and returns nothing. + + """ + + if not path: + path = self.path + self._push_url = None + self._branch_remote = None + # construct the git url + if anon: + giturl = self._get_namespace_anongiturl(module) + else: + giturl = self._get_namespace_giturl(module) + + # Create the command + cmd = ['git', 'clone'] + if self.quiet: + cmd.append('-q') + # do the clone + if branch and bare_dir: + raise rpkgError('Cannot combine bare cloning with a branch') + elif branch: + # For now we have to use switch branch + self.log.debug('Checking out a specific branch %s', giturl) + cmd.extend(['-b', branch, giturl]) + elif bare_dir: + self.log.debug('Cloning %s bare', giturl) + cmd.extend(['--bare', giturl]) + if not target: + cmd.append(bare_dir) + else: + self.log.debug('Cloning %s', giturl) + cmd.extend([giturl]) + + if not bare_dir: + # --bare and --origin are incompatible + cmd.extend(['--origin', self.default_branch_remote]) + + if target: + self.log.debug('Cloning into: %s', target) + cmd.append(target) + + self._run_command(cmd, cwd=path) + + if self.clone_config: + base_module = self.get_base_module(module) + git_dir = target if target else bare_dir if bare_dir else base_module + conf_git = git.Git(os.path.join(path, git_dir)) + self._clone_config(conf_git, module) + + return + + def get_base_module(self, module): + # Handle namespaced modules + # Example: + # module: docker/cockpit + # The path will just be os.path.join(path, "cockpit") + if "/" in module: + return module.split("/")[-1] + return module + + def clone_with_dirs(self, module, anon=False, target=None): + """Clone a repo old style with subdirs for each branch. + + module is the name of the module to clone + + gitargs is an option list of arguments to git clone + + """ + + self._push_url = None + self._branch_remote = None + # Get the full path of, and git object for, our directory of branches + top_path = os.path.join(self.path, + target or self.get_base_module(module)) + top_git = git.Git(top_path) + repo_path = os.path.join(top_path, 'rpkg.git') + + # construct the git url + if anon: + giturl = self._get_namespace_anongiturl(module) + else: + giturl = self._get_namespace_giturl(module) + + # Create our new top directory + try: + os.mkdir(top_path) + except OSError as e: + raise rpkgError('Could not create directory for module %s: %s' + % (module, e)) + + # Create a bare clone first. This gives us a good list of branches + try: + self.clone(module, top_path, bare_dir=repo_path, anon=anon) + except Exception as e: + # Clean out our directory + shutil.rmtree(top_path) + raise + # Get the full path to, and a git object for, our new bare repo + repo_git = git.Git(repo_path) + + # Get a branch listing + branches = [x for x in repo_git.branch().split() + if x != "*" and re.search(self.branchre, x)] + + for branch in branches: + try: + # Make a local clone for our branch + top_git.clone("--branch", branch, + "--origin", self.default_branch_remote, + repo_path, branch) + + # Set the origin correctly + branch_path = os.path.join(top_path, branch) + branch_git = git.Git(branch_path) + branch_git.config("--replace-all", + "remote.%s.url" % self.default_branch_remote, + giturl) + except (git.GitCommandError, OSError) as e: + raise rpkgError('Could not locally clone %s from %s: %s' + % (branch, repo_path, e)) + + # We don't need this now. Ignore errors since keeping it does no harm + shutil.rmtree(repo_path, ignore_errors=True) + + def _clone_config(self, conf_git, module): + clone_config = self.clone_config.strip() % {'module': module} + for confline in clone_config.splitlines(): + if confline: + conf_git.config(*confline.split()) + + def commit(self, message=None, file=None, files=[], signoff=False): + """Commit changes to a module (optionally found at path) + + Can take a message to use as the commit message + + a file to find the commit message within + + and a list of files to commit. + + Requires the caller be a real tty or a message passed. + + Logs the output and returns nothing. + + """ + + # First lets see if we got a message or we're on a real tty: + if not sys.stdin.isatty(): + if not message and not file: + raise rpkgError('Must have a commit message or be on a real ' + 'tty.') + + # construct the git command + # We do this via subprocess because the git module is terrible. + cmd = ['git', 'commit'] + if signoff: + cmd.append('-s') + if self.quiet: + cmd.append('-q') + if message: + cmd.extend(['-m', message]) + elif file: + # If we get a relative file name, prepend our path to it. + if self.path and not file.startswith('/'): + cmd.extend(['-F', os.path.abspath(os.path.join(self.path, + file))]) + else: + cmd.extend(['-F', os.path.abspath(file)]) + if not files: + cmd.append('-a') + else: + cmd.extend(files) + # make it so + self._run_command(cmd, cwd=self.path) + return + + def delete_tag(self, tagname): + """Delete a git tag from the repository found at optional path""" + + try: + self.repo.delete_tag(tagname) + + except git.GitCommandError as e: + raise rpkgError(e) + + self.log.info('Tag %s was deleted', tagname) + + def diff(self, cached=False, files=[]): + """Execute a git diff + + optionally diff the cached or staged changes + + Takes an optional list of files to diff relative to the module base + directory + + Logs the output and returns nothing + + """ + + # Things work better if we're in our module directory + oldpath = os.getcwd() + os.chdir(self.path) + # build up the command + cmd = ['git', 'diff'] + if cached: + cmd.append('--cached') + if files: + cmd.extend(files) + + # Run it! + self._run_command(cmd) + # popd + os.chdir(oldpath) + return + + def get_latest_commit(self, module, branch): + """Discover the latest commit has for a given module and return it""" + + # This is stupid that I have to use subprocess :/ + url = self._get_namespace_anongiturl(module) + # This cmd below only works to scratch build rawhide + # We need something better for epel + cmd = ['git', 'ls-remote', url, 'refs/heads/%s' % branch] + try: + proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, + stdout=subprocess.PIPE) + output, error = proc.communicate() + except OSError as e: + raise rpkgError(e) + if error: + raise rpkgError('Got an error finding %s head for %s: %s' + % (branch, module, error)) + # Return the hash sum + if not output: + raise rpkgError('Could not find remote branch %s for %s' + % (branch, module)) + return output.split()[0] + + def gitbuildhash(self, build): + """Determine the git hash used to produce a particular N-V-R""" + + # Get the build data from the nvr + self.log.debug('Getting task data from the build system') + bdata = self.anon_kojisession.getBuild(build) + if not bdata: + raise rpkgError('Unknown build: %s' % build) + + # Get the task data out of that build data + taskinfo = self.anon_kojisession.getTaskRequest(bdata['task_id']) + # taskinfo is a list of items, first item is the task url. + # second is the build target. + # See if the build target starts with cvs or git + hash = None + buildsource = taskinfo[0] + if buildsource.startswith('cvs://'): + # snag everything after the last # mark + cvstag = buildsource.rsplit('#')[-1] + # Now read the remote repo to figure out the hash from the tag + giturl = self._get_namespace_anongiturl(bdata['name']) + cmd = ['git', 'ls-remote', '--tags', giturl, cvstag] + self.log.debug('Querying git server for tag info') + try: + output = subprocess.check_output(cmd) + hash = output.split()[0] + except: + # don't do anything here, we'll handle not having hash + # later + pass + elif buildsource.startswith('git://'): + # Match a 40 char block of text on the url line, that'll be + # our hash + hash = buildsource.rsplit('#')[-1] + else: + # Unknown build source + raise rpkgError('Unhandled build source %s' % buildsource) + if not hash: + raise rpkgError('Could not find hash of build %s' % build) + return (hash) + + def import_srpm(self, srpm): + """Import the contents of an srpm into a repo. + + srpm: File to import contents from + + This function will add/remove content to match the srpm, + + upload new files to the lookaside, and stage the changes. + + Returns a list of files to upload. + + """ + + # see if the srpm even exists + srpm = os.path.abspath(srpm) + if not os.path.exists(srpm): + raise rpkgError('File not found.') + # bail if we're dirty + if self.repo.is_dirty(): + raise rpkgError('There are uncommitted changes in your repo') + # Get the details of the srpm + name, files, uploadfiles = self._srpmdetails(srpm) + + # Need a way to make sure the srpm name matches the repo some how. + + # Get a list of files we're currently tracking + ourfiles = self.repo.git.ls_files().split('\n') + if ourfiles == ['']: + # Repository doesn't contain any files + ourfiles = [] + else: + # Trim out sources and .gitignore + for file in ('.gitignore', 'sources'): + try: + ourfiles.remove(file) + except ValueError: + pass + + # Things work better if we're in our module directory + oldpath = os.getcwd() + os.chdir(self.path) + + # Look through our files and if it isn't in the new files, remove it. + for file in ourfiles: + if file not in files: + self.log.info("Removing no longer used file: %s", file) + self.repo.index.remove([file]) + os.remove(file) + + # Extract new files + cmd = ['rpm2cpio', srpm] + # We have to force cpio to copy out (u) because git messes with + # timestamps + cmd2 = ['cpio', '-iud', '--quiet'] + + rpmcall = subprocess.Popen(cmd, stdout=subprocess.PIPE) + cpiocall = subprocess.Popen(cmd2, stdin=rpmcall.stdout) + output, err = cpiocall.communicate() + if output: + self.log.debug(output) + if err: + os.chdir(oldpath) + raise rpkgError("Got an error from rpm2cpio: %s" % err) + + # And finally add all the files we know about (and our stock files) + for file in ('.gitignore', 'sources'): + if not os.path.exists(file): + # Create the file + open(file, 'w').close() + files.append(file) + self.repo.index.add(files) + # Return to the caller and let them take it from there. + os.chdir(oldpath) + return(uploadfiles) + + def list_tag(self, tagname='*'): + """List all tags in the repository which match a given tagname. + + The optional `tagname` argument may be a shell glob (it is matched + with fnmatch). + + """ + if tagname is None: + tagname = '*' + + tags = map(lambda t: t.name, self.repo.tags) + + if tagname != '*': + tags = filter(lambda t: fnmatch.fnmatch(t, tagname), tags) + + for tag in tags: + print(tag) + + def new(self): + """Return changes in a repo since the last tag""" + + # Find the latest tag + try: + tag = self.repo.git.describe('--tags', '--abbrev=0') + except git.exc.GitCommandError: + raise rpkgError('Cannot get changes because there are no tags in this repo.') + # Now get the diff + self.log.debug('Diffing from tag %s', tag) + return self.repo.git.diff('-M', tag) + + def patch(self, suffix, rediff=False): + """Generate a patch from the expanded source and add it to index + + suffix: Look for files named with this suffix to diff + rediff: optionally retain any comments in the patch file and rediff + + Will create a patch file named name-version-suffix.patch + """ + + # Create the outfile name based on arguments + outfile = '%s-%s-%s.patch' % (self.module_name, self.ver, suffix) + + # If we want to rediff, the patch file has to already exist + if rediff and not os.path.exists(os.path.join(self.path, outfile)): + raise rpkgError('Patch file %s not found, unable to rediff' % + os.path.join(self.path, outfile)) + + # See if there is a source dir to diff in + if not os.path.isdir(os.path.join(self.path, + '%s-%s' % (self.module_name, + self.ver))): + raise rpkgError('Expanded source dir not found!') + + # Setup the command + cmd = ['gendiff', '%s-%s' % (self.module_name, self.ver), + '.%s' % suffix] + + # Try to run the command and capture the output + try: + self.log.debug('Running %s', ' '.join(cmd)) + (output, errors) = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.path).communicate() + except Exception as e: + raise rpkgError('Error running gendiff: %s' % e) + + # log any errors + if errors: + self.log.error(errors) + + # See if we got anything + if not output: + raise rpkgError('gendiff generated an empty patch!') + + # See if we are rediffing and handle the old patch file + if rediff: + oldpatch = open(os.path.join(self.path, outfile), 'r').readlines() + # back up the old file + self.log.debug('Moving existing patch %s to %s~', outfile, outfile) + os.rename(os.path.join(self.path, outfile), + '%s~' % os.path.join(self.path, outfile)) + # Capture the lines preceding the diff + newhead = [] + for line in oldpatch: + if line.startswith('diff'): + break + else: + newhead.append(line) + + log.debug('Saved from previous patch: \n%s' % ''.join(newhead)) + # Stuff the new head in front of the existing output + output = ''.join(newhead) + output + + # Write out the patch + open(os.path.join(self.path, outfile), 'w').write(output) + + # Add it to the index + # Again this returns a blank line we want to keep quiet + self.repo.index.add([outfile]) + log.info('Created %s and added it to the index' % outfile) + + def pull(self, rebase=False, norebase=False): + """Pull changes from the remote repository + + Optionally rebase current branch on top of remote branch + + Optionally override .git setting to always rebase + + """ + + cmd = ['git', 'pull'] + if self.quiet: + cmd.append('-q') + if rebase: + cmd.append('--rebase') + if norebase: + cmd.append('--no-rebase') + self._run_command(cmd, cwd=self.path) + return + + def find_untracked_patches(self): + """Find patches that are not tracked by git and sources both""" + file_pattern = os.path.join(self.path, '*.patch') + patches_in_repo = [os.path.basename(filename) for filename + in glob.glob(file_pattern)] + + git_tree = self.repo.head.commit.tree + sources_file = SourcesFile(self.sources_filename, + self.source_entry_type) + + patches_not_tracked = [ + patch for patch in patches_in_repo + if patch not in git_tree and patch not in sources_file] + + return patches_not_tracked + + def push(self, force=False): + """Push changes to the remote repository""" + + # see if our branch is tracking anything + try: + self.load_branch_merge() + except: + self.log.warning('Current branch cannot be pushed anywhere!') + + untracked_patches = self.find_untracked_patches() + if untracked_patches: + self.log.warning( + 'Patches %s %s not tracked within either git or sources', + ', '.join(untracked_patches), + 'is' if len(untracked_patches) == 1 else 'are') + + cmd = ['git', 'push'] + if self.quiet: + cmd.append('-q') + self._run_command(cmd, cwd=self.path) + + def sources(self, outdir=None): + """Download source files""" + + if not os.path.exists(self.sources_filename): + self.log.info("sources file doesn't exist. Source files download skipped.") + return + + # Default to putting the files where the module is + if not outdir: + outdir = self.path + + sourcesf = SourcesFile(self.sources_filename, self.source_entry_type) + + for entry in sourcesf.entries: + outfile = os.path.join(outdir, entry.file) + self.lookasidecache.download( + self.module_name, entry.file, entry.hash, outfile, + hashtype=entry.hashtype, branch=self.branch_merge) + + def switch_branch(self, branch, fetch=True): + """Switch the working branch + + Will create a local branch if one doesn't already exist, + based on / + + Logs output and returns nothing. + """ + + # Currently this just grabs the first matching branch name from + # the first remote it finds. When multiple remotes are in play + # this needs to get smarter + + # See if the repo is dirty first + if self.repo.is_dirty(): + raise rpkgError('%s has uncommitted changes. Use git status ' + 'to see details' % self.path) + + # Get our list of branches + (locals, remotes) = self._list_branches(fetch) + + if branch not in locals: + # We need to create a branch + self.log.debug('No local branch found, creating a new one') + totrack = None + full_branch = '%s/%s' % (self.branch_remote, branch) + for remote in remotes: + if remote == full_branch: + totrack = remote + break + else: + raise rpkgError('Unknown remote branch %s' % full_branch) + try: + self.log.info(self.repo.git.checkout('-b', branch, '--track', totrack)) + except Exception as err: + # This needs to be finer grained I think... + raise rpkgError('Could not create branch %s: %s' + % (branch, err)) + else: + try: + self.repo.git.checkout(branch) + # The above should have no output, but stash it anyway + self.log.info("Switched to branch '%s'", branch) + except Exception as err: + # This needs to be finer grained I think... + raise rpkgError('Could not check out %s\n%s' % (branch, + err.stderr)) + return + + def check_repo(self, is_dirty=True, all_pushed=True): + if is_dirty: + if self.repo.is_dirty(): + raise rpkgError('%s has uncommitted changes. Use git status ' + 'to see details' % self.path) + if all_pushed: + branch = self.repo.active_branch + remote = self.repo.git.config('--get', 'branch.%s.remote' % branch) + merge = self.repo.git.config('--get', 'branch.%s.merge' % branch).replace('refs/heads', + remote) + if self.repo.git.rev_list('%s...%s' % (merge, branch)): + raise rpkgError('There are unpushed changes in your repo') + + def build(self, skip_tag=False, scratch=False, background=False, + url=None, chain=None, arches=None, sets=False, nvr_check=True): + """Initiate a build of the module. Available options are: + + skip_tag: Skip the tag action after the build + + scratch: Perform a scratch build + + background: Perform the build with a low priority + + url: A url to an uploaded srpm to build from + + chain: A chain build set + + arches: A set of arches to limit the scratch build for + + sets: A boolean to let us know whether or not the chain has sets + + nvr_check: A boolean; locally construct NVR and submit a build only if + NVR doesn't exist in a build system + + This function submits the task to koji and returns the taskID + + It is up to the client to wait or watch the task. + """ + + # Ensure the repo exists as well as repo data and site data + # build up the command that a user would issue + cmd = [self.build_client] + # construct the url + if not url: + # We don't have a url, so build from the latest commit + # Check to see if the tree is dirty and if all local commits + # are pushed + self.check_repo() + url = self._get_namespace_anongiturl(self.ns_module_name) + \ + '?#%s' % self.commithash + # Check to see if the target is valid + build_target = self.kojisession.getBuildTarget(self.target) + if not build_target: + raise rpkgError('Unknown build target: %s' % self.target) + # see if the dest tag is locked + dest_tag = self.kojisession.getTag(build_target['dest_tag_name']) + if not dest_tag: + raise rpkgError('Unknown destination tag %s' + % build_target['dest_tag_name']) + if dest_tag['locked'] and not scratch: + raise rpkgError('Destination tag %s is locked' % dest_tag['name']) + # If we're chain building, make sure inheritance works + if chain: + cmd.append('chain-build') + ancestors = self.kojisession.getFullInheritance( + build_target['build_tag']) + ancestors = [ancestor['parent_id'] for ancestor in ancestors] + if dest_tag['id'] not in [build_target['build_tag']] + ancestors: + raise rpkgError('Packages in destination tag ' + '%(dest_tag_name)s are not inherited by' + 'build tag %(build_tag_name)s' % + build_target) + else: + cmd.append('build') + # define our dictionary for options + opts = {} + # Set a placeholder for the build priority + priority = None + if skip_tag: + opts['skip_tag'] = True + cmd.append('--skip-tag') + if scratch: + opts['scratch'] = True + cmd.append('--scratch') + if background: + cmd.append('--background') + priority = 5 # magic koji number :/ + if arches: + if not scratch: + raise rpkgError('Cannot override arches for non-scratch ' + 'builds') + for arch in arches: + if not re.match(r'^[0-9a-zA-Z_.]+$', arch): + raise rpkgError('Invalid architecture name: %s' % arch) + cmd.append('--arch-override=%s' % ','.join(arches)) + opts['arch_override'] = ' '.join(arches) + + cmd.append(self.target) + + if url.endswith('.src.rpm'): + srpm = os.path.basename(url) + build_reference = srpm + else: + try: + build_reference = self.nvr + except rpkgError as error: + self.log.warning(error) + if nvr_check: + self.log.info('Note: You can skip NVR construction & NVR' + ' check with --skip-nvr-check. See help for' + ' more info.') + raise rpkgError('Cannot continue without properly constructed NVR.') + else: + self.log.info('NVR checking will be skipped so I do not' + ' care that I am not able to construct NVR.' + ' I will refer this build by package name' + ' in following messages.') + build_reference = self.module_name + + # see if this build has been done. Does not check builds within + # a chain + if nvr_check and not scratch and not url.endswith('.src.rpm'): + build = self.kojisession.getBuild(self.nvr) + if build: + if build['state'] == 1: + raise rpkgError('Package %s has already been built\n' + 'Note: You can skip this check with' + ' --skip-nvr-check. See help for more' + ' info.' % self.nvr) + # Now submit the task and get the task_id to return + # Handle the chain build version + if chain: + self.log.debug('Adding %s to the chain', url) + # If we're dealing with build sets the behaviour of the last + # package changes, and we add it to the last (potentially empty) + # set. Otherwise the last package just gets added to the end of + # the chain. + if sets: + chain[-1].append(url) + else: + chain.append([url]) + # This next list comp is ugly, but it's how we properly get a : + # put in between each build set + cmd.extend(' : '.join([' '.join(build_sets) for build_sets in chain]).split()) + self.log.info('Chain building %s + %s for %s', build_reference, chain[:-1], self.target) + self.log.debug('Building chain %s for %s with options %s and a priority of %s', + chain, self.target, opts, priority) + self.log.debug(' '.join(cmd)) + task_id = self.kojisession.chainBuild(chain, self.target, opts, priority=priority) + # Now handle the normal build + else: + cmd.append(url) + self.log.info('Building %s for %s', build_reference, self.target) + self.log.debug('Building %s for %s with options %s and a priority of %s', + url, self.target, opts, priority) + self.log.debug(' '.join(cmd)) + task_id = self.kojisession.build(url, self.target, opts, priority=priority) + self.log.info('Created task: %s', task_id) + self.log.info('Task info: %s/taskinfo?taskID=%s', self.kojiweburl, task_id) + return task_id + + def clog(self, raw=False): + """Write the latest spec changelog entry to a clog file""" + + # This is a little ugly. We want to find where %changelog starts, + # then only deal with the content up to the first empty newline. + # Then remove any lines that start with $ or %, and then replace + # %% with % + + cloglines = [] + first = True + spec = open(os.path.join(self.path, self.spec), 'r').readlines() + for line in spec: + if line.lower().startswith('%changelog'): + # Grab all the lines below changelog + for line2 in spec[spec.index(line):]: + if line2.startswith('\n'): + break + if line2.startswith('$'): + continue + if line2.startswith('%'): + continue + if line2.startswith('*'): + if first: + # skip the email n/v/r line. Redundant + continue + # Otherwise what follows is the next entry + break + if first: + if not raw: + cloglines.append(line2.lstrip('- ').replace('%%', + '%')) + cloglines.append("\n") + else: + cloglines.append(line2.replace('%%', '%')) + first = False + else: + cloglines.append(line2.replace('%%', '%')) + + # Now open the clog file and write out the lines + clogfile = open(os.path.join(self.path, 'clog'), 'w') + clogfile.writelines(cloglines) + + def compile(self, arch=None, short=False, builddir=None, nocheck=False): + """Run rpmbuild -bc on a module + + optionally for a specific arch, or short-circuit it, or + define an alternate builddir + + Logs the output and returns nothing + """ + + # Get the sources + self.sources() + # setup the rpm command + cmd = ['rpmbuild'] + if builddir: + # Tack on a new builddir to the end of the defines + self.rpmdefines.append("--define '_builddir %s'" % + os.path.abspath(builddir)) + cmd.extend(self.rpmdefines) + if arch: + cmd.extend(['--target', arch]) + if short: + cmd.append('--short-circuit') + if nocheck: + cmd.append('--nocheck') + if self.quiet: + cmd.append('--quiet') + cmd.extend(['-bc', os.path.join(self.path, self.spec)]) + # Run the command + self._run_command(cmd, shell=True) + + def giturl(self): + """Return the git url that would be used for building""" + + url = self._get_namespace_anongiturl(self.ns_module_name) + \ + '?#%s' % self.commithash + return url + + def koji_upload(self, file, path, callback=None): + """Upload a file to koji + + file is the file you wish to upload + + path is the relative path on the server to upload to + + callback is the progress callback to use, if any + + Returns nothing or raises + """ + + # See if we actually have a file + if not os.path.exists(file): + raise rpkgError('No such file: %s' % file) + if not self.kojisession: + raise rpkgError('No active %s session.' % + os.path.basename(self.build_client)) + # This should have a try and catch koji errors + self.kojisession.uploadWrapper(file, path, callback=callback) + + def install(self, arch=None, short=False, builddir=None, nocheck=False): + """Run rpm -bi on a module + + optionally for a specific arch, short-circuit it, or + define an alternative builddir + + Logs the output and returns nothing + """ + + # Get the sources + self.sources() + # setup the rpm command + cmd = ['rpmbuild'] + if builddir: + # Tack on a new builddir to the end of the defines + self.rpmdefines.append("--define '_builddir %s'" % + os.path.abspath(builddir)) + cmd.extend(self.rpmdefines) + if arch: + cmd.extend(['--target', arch]) + if short: + cmd.append('--short-circuit') + if nocheck: + cmd.append('--nocheck') + if self.quiet: + cmd.append('--quiet') + cmd.extend(['-bi', os.path.join(self.path, self.spec)]) + # Run the command + self._run_command(cmd, shell=True) + return + + def lint(self, info=False, rpmlintconf=None): + """Run rpmlint over a built srpm + + Log the output and returns nothing + rpmlintconf is the name of the config file passed to rpmlint if + specified by the command line argument. + """ + + # Check for srpm + srpm = "%s-%s-%s.src.rpm" % (self.module_name, self.ver, self.rel) + if not os.path.exists(os.path.join(self.path, srpm)): + log.warning('No srpm found') + + # Get the possible built arches + arches = self._get_build_arches_from_spec() + rpms = [] + for arch in arches: + if os.path.exists(os.path.join(self.path, arch)): + # For each available arch folder, lists file and keep + # those ending with .rpm + rpms.extend([os.path.join(self.path, arch, file) + for file in os.listdir(os.path.join(self.path, + arch)) + if file.endswith('.rpm')]) + if not rpms: + log.warning('No rpm found') + cmd = ['rpmlint'] + if info: + cmd.extend(['-i']) + if rpmlintconf: + cmd.extend(["-f", os.path.join(self.path, rpmlintconf)]) + elif os.path.exists(os.path.join(self.path, ".rpmlint")): + cmd.extend(["-f", os.path.join(self.path, ".rpmlint")]) + cmd.append(os.path.join(self.path, self.spec)) + if os.path.exists(os.path.join(self.path, srpm)): + cmd.append(os.path.join(self.path, srpm)) + cmd.extend(rpms) + # Run the command + self._run_command(cmd, shell=True) + + def local(self, arch=None, hashtype=None, builddir=None): + """rpmbuild locally for given arch. + + Takes arch to build for, and hashtype to build with. + + Writes output to a log file and logs it to the logger + + Returns the returncode from the build call + """ + + # This could really use a list of arches to build for and loop over + # Get the sources + self.sources() + # build up the rpm command + cmd = ['rpmbuild'] + if builddir: + # Tack on a new builddir to the end of the defines + self.rpmdefines.append("--define '_builddir %s'" % + os.path.abspath(builddir)) + cmd.extend(self.rpmdefines) + # Figure out the hash type to use + if not hashtype: + # Try to determine the dist + hashtype = self._guess_hashtype() + # This may need to get updated if we ever change our checksum default + if not hashtype == 'sha256': + cmd.extend(["--define '_source_filedigest_algorithm %s'" + % hashtype, + "--define '_binary_filedigest_algorithm %s'" + % hashtype]) + if arch: + cmd.extend(['--target', arch]) + if self.quiet: + cmd.append('--quiet') + cmd.extend(['-ba', os.path.join(self.path, self.spec)]) + logfile = '.build-%s-%s.log' % (self.ver, self.rel) + # Run the command + self._run_command(cmd, shell=True, pipe=['tee', logfile]) + + # Not to be confused with mockconfig the property + def mock_config(self, target=None, arch=None): + """Generate a mock config based on branch data. + + Can use option target and arch to override autodiscovery. + Will return the mock config file text. + """ + + # Figure out some things about ourself. + if not target: + target = self.target + if not arch: + arch = self.localarch + + # Figure out if we have a valid build target + build_target = self.anon_kojisession.getBuildTarget(target) + if not build_target: + raise rpkgError('Unknown build target: %s\n' + 'Consider using the --target option' % target) + + try: + repoid = self.anon_kojisession.getRepo( + build_target['build_tag_name'])['id'] + except Exception: + raise rpkgError('Could not find a valid build repo') + + # Generate the config + config = koji.genMockConfig('%s-%s' % (target, arch), arch, + distribution=self.disttag, + tag_name=build_target['build_tag_name'], + repoid=repoid, + topurl=self.topurl) + + # Return the mess + return(config) + + def _config_dir_other(self, config_dir, filenames=('site-defaults.cfg', + 'logging.ini')): + """Populates mock config directory with other necessary files + + If files are found in system config directory for mock they are copied + to mock config directory defined as method's argument. Otherwise empty + files are created.""" + for filename in filenames: + system_filename = '/etc/mock/%s' % filename + tmp_filename = os.path.join(config_dir, filename) + if os.path.exists(system_filename): + try: + shutil.copy2(system_filename, tmp_filename) + except Exception as error: + raise rpkgError('Failed to create copy system config file' + ' %s: %s' % (filename, error)) + else: + try: + open(tmp_filename, 'w').close() + except Exception as error: + raise rpkgError('Failed to create empty mock config' + ' file %s: %s' + % (tmp_filename, error)) + + def _config_dir_basic(self, config_dir=None, root=None): + """Setup directory with essential mock config + + If config directory doesn't exist it will be created. If temporary + directory was created by this method and error occours during + processing, temporary directory is removed. Otherwise it caller's + responsibility to remove this directory. + + Returns used config directory""" + if not root: + root = self.mockconfig + if not config_dir: + my_config_dir = tempfile.mkdtemp(prefix="%s." % root, + suffix='mockconfig') + config_dir = my_config_dir + self.log.debug('New mock config directory: %s', config_dir) + else: + my_config_dir = None + + try: + config_content = self.mock_config() + except rpkgError as error: + self._cleanup_tmp_dir(my_config_dir) + raise rpkgError('Could not generate config file: %s' + % error) + + config_file = os.path.join(config_dir, '%s.cfg' % root) + try: + open(config_file, 'wb').write(config_content) + except IOError as error: + self._cleanup_tmp_dir(my_config_dir) + raise rpkgError('Could not write config file: %s' % error) + + return config_dir + + def _cleanup_tmp_dir(self, tmp_dir): + """Tries to remove directory and ignores EEXIST error + + If occoured directory not exist error (EEXIST) it silently continue. + Otherwise raise rpkgError exception.""" + if not tmp_dir: + return + try: + shutil.rmtree(tmp_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise rpkgError('Failed to remove temporary directory' + ' %s. Reason: %s.' % (tmp_dir, error)) + + def mockbuild(self, mockargs=[], root=None, hashtype=None): + """Build the package in mock, using mockargs + + Log the output and returns nothing + """ + + # Make sure we have an srpm to run on + self.srpm(hashtype=hashtype) + + # setup the command + cmd = ['mock'] + cmd.extend(mockargs) + if self.quiet: + cmd.append('--quiet') + + config_dir = None + if not root: + root = self.mockconfig + chroot_cfg = '/etc/mock/%s.cfg' % root + if not os.path.exists(chroot_cfg): + self.log.debug('Mock config %s was not found. Going to' + ' request koji to create new one.', chroot_cfg) + try: + config_dir = self._config_dir_basic(root=root) + except rpkgError as error: + raise rpkgError('Failed to create mock config directory:' + ' %s' % error) + self.log.debug('Temporary mock config directory: %s', config_dir) + try: + self._config_dir_other(config_dir) + except rpkgError as error: + self._cleanup_tmp_dir(config_dir) + raise rpkgError('Failed to populate mock config directory:' + ' %s' % error) + cmd.extend(['--configdir', config_dir]) + + cmd.extend(['-r', root, '--resultdir', self.mock_results_dir, + '--rebuild', self.srpmname]) + # Run the command + try: + self._run_command(cmd) + finally: + self.log.debug('Cleaning up mock temporary config directory: %s', config_dir) + self._cleanup_tmp_dir(config_dir) + + def upload(self, files, replace=False): + """Upload source file(s) in the lookaside cache + + Can optionally replace the existing tracked sources + """ + + sourcesf = SourcesFile(self.sources_filename, self.source_entry_type, + replace=replace) + gitignore = GitIgnore(os.path.join(self.path, '.gitignore')) + + for f in files: + # TODO: Skip empty file needed? + file_hash = self.lookasidecache.hash_file(f) + file_basename = os.path.basename(f) + + try: + sourcesf.add_entry(self.lookasidehash, file_basename, + file_hash) + except HashtypeMixingError as e: + msg = '\n'.join([ + 'Can not upload a new source file with a %(newhash)s ' + 'hash, as the "%(sources)s" file contains at least one ' + 'line with a %(existinghash)s hash.', '', + 'Please redo the whole "%(sources)s" file using:', + ' `%(arg0)s new-sources file1 file2 ...`']) % { + 'newhash': e.new_hashtype, + 'existinghash': e.existing_hashtype, + 'sources': self.sources_filename, + 'arg0': sys.argv[0], + } + raise rpkgError(msg) + + gitignore.add('/%s' % file_basename) + self.lookasidecache.upload(self.module_name, f, file_hash) + + sourcesf.write() + gitignore.write() + + self.repo.index.add(['sources', '.gitignore']) + + def prep(self, arch=None, builddir=None): + """Run rpm -bp on a module + + optionally for a specific arch, or + define an alternative builddir + + Logs the output and returns nothing + """ + + # Get the sources + self.sources() + # setup the rpm command + cmd = ['rpmbuild'] + if builddir: + # Tack on a new builddir to the end of the defines + self.rpmdefines.append("--define '_builddir %s'" % + os.path.abspath(builddir)) + cmd.extend(self.rpmdefines) + if arch: + cmd.extend(['--target', arch]) + if self.quiet: + cmd.append('--quiet') + cmd.extend(['--nodeps', '-bp', os.path.join(self.path, self.spec)]) + # Run the command + self._run_command(cmd, shell=True) + + def srpm(self, hashtype=None): + """Create an srpm using hashtype from content in the module + + Requires sources already downloaded. + """ + + self.srpmname = os.path.join(self.path, + "%s-%s-%s.src.rpm" + % (self.module_name, self.ver, self.rel)) + + # See if we need to build the srpm + if os.path.exists(self.srpmname): + self.log.debug('Srpm found, rewriting it.') + + cmd = ['rpmbuild'] + cmd.extend(self.rpmdefines) + if self.quiet: + cmd.append('--quiet') + # Figure out which hashtype to use, if not provided one + if not hashtype: + # Try to determine the dist + hashtype = self._guess_hashtype() + # This may need to get updated if we ever change our checksum default + if not hashtype == 'sha256': + cmd.extend(["--define '_source_filedigest_algorithm %s'" + % hashtype, + "--define '_binary_filedigest_algorithm %s'" + % hashtype]) + cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)]) + self._run_command(cmd, shell=True) + + def unused_patches(self): + """Discover patches checked into source control that are not used + + Returns a list of unused patches, which may be empty. + """ + + # Create a list for unused patches + unused = [] + # Get the content of spec into memory for fast searching + with open(self.spec, 'r') as f: + data = f.read() + try: + spec = data.decode('UTF-8') + except UnicodeDecodeError as error: + # when can't decode file, ignore chars and show warning + spec = data.decode('UTF-8', 'ignore') + line, offset = self._byte_offset_to_line_number(spec, error.start) + self.log.warning("'%s' codec can't decode byte in position %d:%d : %s", + error.encoding, line, offset, error.reason) + # Replace %{name} with the package name + spec = spec.replace("%{name}", self.module_name) + # Replace %{version} with the package version + spec = spec.replace("%{version}", self.ver) + + # Get a list of files tracked in source control + files = self.repo.git.ls_files('--exclude-standard').split() + for file in files: + # throw out non patches + if not file.endswith(('.patch', '.diff')): + continue + if file not in spec: + unused.append(file) + return unused + + def _byte_offset_to_line_number(self, text, offset): + """ + Convert byte offset (given by e.g. DecodeError) to human readable + format (line number and char position) + Return a list with line number and char offset + """ + offset_inc = 0 + line_num = 1 + for line in text.split('\n'): + if offset_inc + len(line) + 1 > offset: + break + else: + offset_inc += len(line) + 1 + line_num += 1 + return [line_num, offset - offset_inc + 1] + + def verify_files(self, builddir=None): + """Run rpmbuild -bl on a module to verify the %files section + + optionally define an alternate builddir + """ + + # setup the rpm command + cmd = ['rpmbuild'] + if builddir: + # Tack on a new builddir to the end of the defines + self.rpmdefines.append("--define '_builddir %s'" % + os.path.abspath(builddir)) + cmd.extend(self.rpmdefines) + if self.quiet: + cmd.append('--quiet') + cmd.extend(['-bl', os.path.join(self.path, self.spec)]) + # Run the command + self._run_command(cmd, shell=True) + + def osbs_build(self, config_file, config_section, target_override=False, + yum_repourls=[], nowait=False): + self.check_repo() + os_conf = Configuration(conf_file=config_file, conf_section=config_section) + build_conf = Configuration(conf_file=config_file, conf_section=config_section) + osbs = OSBS(os_conf, build_conf) + + git_uri = re.sub(r"^git\+ssh", "git", self.push_url) + git_uri = re.sub("^ssh", "git", git_uri) + git_uri = re.sub("[^/]+@", "", git_uri) + git_ref = self.commithash + git_branch = self.branch_merge + user = self.user + component = self.module_name + docker_target = self.target + if not target_override: + # Translate the build target into a docker target, + # but only if --target wasn't specified on the command-line + docker_target = '%s-docker-candidate' % self.target.split('-candidate')[0] + + build = osbs.create_build( + git_uri=git_uri, + git_ref=git_ref, + git_branch=git_branch, + user=user, + component=component, + target=docker_target, + architecture="x86_64", + yum_repourls=yum_repourls + ) + build_id = build.build_id + + if nowait: + self.log.info('Build submitted: %s', build_id) + return + + print("Build submitted (%s), watching logs (feel free to interrupt)" % build_id) + for line in osbs.get_build_logs(build_id, follow=True): + print(line) + build_response = osbs.wait_for_build_to_finish(build_id) + if build_response.is_succeeded(): + repositories = build_response.get_repositories() + if repositories: + image_names = repositories.get("primary", []) + repositories.get("unique", []) + print("You can pull the image with one of the following commands:") + for image in image_names: + print(" docker pull %s" % image) + else: + raise RuntimeError( + "Build '%s' wasn't processed correctly. Please, report this." % build_id) + else: + raise RuntimeError("Build has failed.") + + def container_build_koji(self, target_override=False, opts={}, + kojiconfig=None, build_client=None, + koji_task_watcher=None, + nowait=False): + # check if repo is dirty and all commits are pushed + self.check_repo() + docker_target = self.target + if not target_override: + # Translate the build target into a docker target, + # but only if --target wasn't specified on the command-line + docker_target = '%s-docker-candidate' % self.target.split('-candidate')[0] + + koji_session_backup = (self.build_client, self.kojiconfig) + (self.build_client, self.kojiconfig) = (build_client, kojiconfig) + try: + self.load_kojisession() + if "buildContainer" not in self.kojisession.system.listMethods(): + raise RuntimeError("Kojihub instance does not support buildContainer") + + build_target = self.kojisession.getBuildTarget(docker_target) + if not build_target: + msg = "Unknown build target: %s" % docker_target + self.log.error(msg) + raise UnknownTargetError(msg) + else: + dest_tag = self.kojisession.getTag(build_target['dest_tag']) + if not dest_tag: + self.log.error("Unknown destination tag: %s", build_target['dest_tag_name']) + if dest_tag['locked'] and 'scratch' not in opts: + self.log.error("Destination tag %s is locked", dest_tag['name']) + + source = self._get_namespace_anongiturl(self.ns_module_name) + source += "#%s" % self.commithash + + task_opts = {} + for key in ('scratch', 'name', 'version', 'release', + 'yum_repourls', 'git_branch'): + if key in opts: + task_opts[key] = opts[key] + priority = opts.get("priority", None) + task_id = self.kojisession.buildContainer(source, + docker_target, + task_opts, + priority=priority) + self.log.info('Created task: %s', task_id) + self.log.info('Task info: %s/taskinfo?taskID=%s', self.kojiweburl, task_id) + if not nowait: + rv = koji_task_watcher(self.kojisession, [task_id]) + if rv == 0: + result = self.kojisession.getTaskResult(task_id) + try: + result["koji_builds"] = [ + "%s/buildinfo?buildID=%s" % (self.kojiweburl, + build_id) + for build_id in result.get("koji_builds", [])] + except TypeError: + pass + log_result(self.log.info, result) + + finally: + (self.build_client, self.kojiconfig) = koji_session_backup + self.load_kojisession() + + def container_build_setup(self, get_autorebuild=None, + set_autorebuild=None): + cfp = ConfigParser.SafeConfigParser() + if os.path.exists(self.osbs_config_filename): + cfp.read(self.osbs_config_filename) + + if get_autorebuild is not None: + if not cfp.has_option('autorebuild', 'enabled'): + self.log.info('true') + else: + self.log.info('true' if cfp.getboolean('autorebuild', 'enabled') else 'false') + elif set_autorebuild is not None: + if not cfp.has_section('autorebuild'): + cfp.add_section('autorebuild') + + cfp.set('autorebuild', 'enabled', set_autorebuild) + with open(self.osbs_config_filename, 'w') as fp: + cfp.write(fp) + + self.repo.index.add([self.osbs_config_filename]) + self.log.info("Config value changed, don't forget to commit %s file", + self.osbs_config_filename) + else: + self.log.info('Nothing to be done') + + def copr_build(self, project, srpm_name, nowait): + cmd = ['copr-cli', 'build'] + if nowait: + cmd.append('--nowait') + cmd.extend([project, srpm_name]) + self._run_command(cmd) diff --git a/pyrpkg/cli.py b/pyrpkg/cli.py new file mode 100755 index 0000000..a48d7c9 --- /dev/null +++ b/pyrpkg/cli.py @@ -0,0 +1,1607 @@ +# cli.py - a cli client class module +# +# Copyright (C) 2011 Red Hat Inc. +# Author(s): Jesse Keating +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. +# +# There are 6 functions derived from /usr/bin/koji which are licensed under +# LGPLv2.1. See comments before those functions. + +import argparse +import sys +import os +import logging +import time +import random +import string +from six.moves import xmlrpc_client +import pwd +import koji + +import utils + +OSBS_DEFAULT_CONF_FILE = "/etc/osbs/osbs.conf" + + +class cliClient(object): + """This is a client class for rpkg clients.""" + + def __init__(self, config, name=None): + """This requires a ConfigParser object + + Name of the app can optionally set, or discovered from exe name + """ + + self.config = config + self._name = name + # Define default name in child class + # self.DEFAULT_CLI_NAME = None + # Property holders, set to none + self._cmd = None + self._module = None + # Setup the base argparser + self.setup_argparser() + # Add a subparser + self.subparsers = self.parser.add_subparsers( + title='Targets', + description='These are valid commands you can ask %s to do' + % self.name) + # Register all the commands + self.setup_subparsers() + + @property + def name(self): + """Property used to identify prog name and key in config file""" + + if not self._name: + self._name = self.get_name() + assert self._name + return(self._name) + + def get_name(self): + name = os.path.basename(sys.argv[0]) + if not name or '__main__.py' in name: + try: + name = self.DEFAULT_CLI_NAME + except AttributeError: + # Ignore missing DEFAULT_CLI_NAME for backwards + # compatibility + pass + if not name: + # We don't have logger available yet + sys.stderr.write('Could not determine CLI name\n') + sys.exit(1) + return name + + # Define some properties here, for lazy loading + @property + def cmd(self): + """This is a property for the command attribute""" + + if not self._cmd: + self.load_cmd() + return(self._cmd) + + def load_cmd(self): + """This sets up the cmd object""" + + # Set target if we got it as an option + target = None + if hasattr(self.args, 'target') and self.args.target: + target = self.args.target + + # load items from the config file + items = dict(self.config.items(self.name, raw=True)) + + dg_namespaced = items.get("distgit_namespaced", False) + + # Create the cmd object + self._cmd = self.site.Commands(self.args.path, + items['lookaside'], + items['lookasidehash'], + items['lookaside_cgi'], + items['gitbaseurl'], + items['anongiturl'], + items['branchre'], + items['kojiconfig'], + items['build_client'], + user=self.args.user, + dist=self.args.dist, + target=target, + quiet=self.args.q, + distgit_namespaced=dg_namespaced + ) + + self._cmd.module_name = self.args.module_name + self._cmd.password = self.args.password + self._cmd.runas = self.args.runas + self._cmd.debug = self.args.debug + self._cmd.verbose = self.args.v + self._cmd.clone_config = items.get('clone_config') + + # This function loads the extra stuff once we figure out what site + # we are + def do_imports(self, site=None): + """Import extra stuff not needed during build + + As a side effect method sets self.site with a loaded library. + + site option can be used to specify which library to load + """ + + # We do some imports here to be more flexible + if not site: + import pyrpkg + self.site = pyrpkg + else: + try: + __import__(site) + self.site = sys.modules[site] + except ImportError: + raise Exception('Unknown site %s' % site) + + def setup_argparser(self): + """Setup the argument parser and register some basic commands.""" + + self.parser = argparse.ArgumentParser( + prog=self.name, + epilog='For detailed help pass --help to a target') + # Add some basic arguments that should be used by all. + # Add a config file + self.parser.add_argument('--config', '-C', + default=None, + help='Specify a config file to use') + # Allow forcing the dist value + self.parser.add_argument('--dist', default=None, + help='Override the discovered distribution') + # Allow forcing the package name + self.parser.add_argument('--module-name', + help=('Override the module name. Otherwise' + ' it is discovered from: Git push URL' + ' or Git URL (last part of path with' + ' .git extension removed) or from name' + ' macro in spec file. In that order.') + ) + # Override the discovered user name + self.parser.add_argument('--user', default=None, + help='Override the discovered user name') + # If using password auth + self.parser.add_argument('--password', default=None, + help='Password for Koji login') + # Run Koji commands as a user other then the one you have + # credentials for (requires configuration on the Koji hub) + self.parser.add_argument('--runas', default=None, + help='Run Koji commands as a different user') + # Let the user define a path to work in rather than cwd + self.parser.add_argument('--path', default=None, + type=utils.u, + help='Define the directory to work in ' + '(defaults to cwd)') + # Verbosity + self.parser.add_argument('--verbose', '-v', dest='v', + action='store_true', + help='Run with verbose debug output') + self.parser.add_argument('--debug', '-d', dest='debug', + action='store_true', + help='Run with debug output') + self.parser.add_argument('-q', action='store_true', + help='Run quietly only displaying errors') + + def setup_subparsers(self): + """Setup basic subparsers that all clients should use""" + + # Setup some basic shared subparsers + + # help command + self.register_help() + + # Add a common parsers + self.register_build_common() + self.register_rpm_common() + + # Other targets + self.register_build() + self.register_chainbuild() + self.register_clean() + self.register_clog() + self.register_clone() + self.register_copr_build() + self.register_commit() + self.register_compile() + self.register_container_build() + self.register_container_build_setup() + self.register_diff() + self.register_gimmespec() + self.register_gitbuildhash() + self.register_giturl() + self.register_import_srpm() + self.register_install() + self.register_lint() + self.register_local() + self.register_mockbuild() + self.register_mock_config() + self.register_new() + self.register_new_sources() + self.register_patch() + self.register_prep() + self.register_pull() + self.register_push() + self.register_scratch_build() + self.register_sources() + self.register_srpm() + self.register_switch_branch() + self.register_tag() + self.register_unused_patches() + self.register_upload() + self.register_verify_files() + self.register_verrel() + + # All the register functions go here. + def register_help(self): + """Register the help command.""" + + help_parser = self.subparsers.add_parser('help', help='Show usage') + help_parser.set_defaults(command=self.parser.print_help) + + # Setup a couple common parsers to save code duplication + def register_build_common(self): + """Create a common build parser to use in other commands""" + + self.build_parser_common = argparse.ArgumentParser( + 'build_common', add_help=False) + self.build_parser_common.add_argument( + '--arches', nargs='*', help='Build for specific arches') + self.build_parser_common.add_argument( + '--md5', action='store_const', const='md5', default=None, + dest='hash', help='Use md5 checksums (for older rpm hosts)') + self.build_parser_common.add_argument( + '--nowait', action='store_true', default=False, + help="Don't wait on build") + self.build_parser_common.add_argument( + '--target', default=None, + help='Define build target to build into') + self.build_parser_common.add_argument( + '--background', action='store_true', default=False, + help='Run the build at a low priority') + + def register_rpm_common(self): + """Create a common parser for rpm commands""" + + self.rpm_parser_common = argparse.ArgumentParser( + 'rpm_common', add_help=False) + self.rpm_parser_common.add_argument( + '--builddir', default=None, help='Define an alternate builddir') + self.rpm_parser_common.add_argument( + '--arch', help='Prep for a specific arch') + + def register_build(self): + """Register the build target""" + + build_parser = self.subparsers.add_parser( + 'build', help='Request build', parents=[self.build_parser_common], + description='This command requests a build of the package in the ' + 'build system. By default it discovers the target ' + 'to build for based on branch data, and uses the ' + 'latest commit as the build source.') + build_parser.add_argument( + '--skip-nvr-check', action='store_false', default=True, + dest='nvr_check', + help='Submit build to buildsystem without check if NVR was ' + 'already build. NVR is constructed locally and may be ' + 'different from NVR constructed during build on builder.') + build_parser.add_argument( + '--skip-tag', action='store_true', default=False, + help='Do not attempt to tag package') + build_parser.add_argument( + '--scratch', action='store_true', default=False, + help='Perform a scratch build') + build_parser.add_argument( + '--srpm', nargs='?', const='CONSTRUCT', + help='Build from an srpm. If no srpm is provided with this option' + ' an srpm will be generated from current module content.') + build_parser.set_defaults(command=self.build) + + def register_chainbuild(self): + """Register the chain build target""" + + chainbuild_parser = self.subparsers.add_parser( + 'chain-build', parents=[self.build_parser_common], + help='Build current package in order with other packages', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +Build current package in order with other packages. + +example: %(name)s chain-build libwidget libgizmo + +The current package is added to the end of the CHAIN list. +Colons (:) can be used in the CHAIN parameter to define groups of +packages. Packages in any single group will be built in parallel +and all packages in a group must build successfully and populate +the repository before the next group will begin building. + +For example: + +%(name)s chain-build libwidget libaselib : libgizmo : + +will cause libwidget and libaselib to be built in parallel, followed +by libgizmo and then the current directory package. If no groups are +defined, packages will be built sequentially.""" % {'name': self.name}) + chainbuild_parser.add_argument( + 'package', nargs='+', + help='List the packages and order you want to build in') + chainbuild_parser.set_defaults(command=self.chainbuild) + + def register_clean(self): + """Register the clean target""" + clean_parser = self.subparsers.add_parser( + 'clean', help='Remove untracked files', + description="This command can be used to clean up your working " + "directory. By default it will follow .gitignore " + "rules.") + clean_parser.add_argument( + '--dry-run', '-n', action='store_true', help='Perform a dry-run') + clean_parser.add_argument( + '-x', action='store_true', help='Do not follow .gitignore rules') + clean_parser.set_defaults(command=self.clean) + + def register_clog(self): + """Register the clog target""" + + clog_parser = self.subparsers.add_parser( + 'clog', help='Make a clog file containing top changelog entry', + description='This will create a file named "clog" that contains ' + 'the latest rpm changelog entry. The leading "- " ' + 'text will be stripped.') + clog_parser.add_argument( + '--raw', action='store_true', default=False, + help='Generate a more "raw" clog without twiddling the contents') + clog_parser.set_defaults(command=self.clog) + + def register_clone(self): + """Register the clone target and co alias""" + + clone_parser = self.subparsers.add_parser( + 'clone', help='Clone and checkout a module', + description='This command will clone the named module from the ' + 'configured repository base URL. By default it will ' + 'also checkout the master branch for your working ' + 'copy.') + # Allow an old style clone with subdirs for branches + clone_parser.add_argument( + '--branches', '-B', action='store_true', + help='Do an old style checkout with subdirs for branches') + # provide a convenient way to get to a specific branch + clone_parser.add_argument( + '--branch', '-b', help='Check out a specific branch') + # allow to clone without needing a account on the scm server + clone_parser.add_argument( + '--anonymous', '-a', action='store_true', + help='Check out a module anonymously') + # store the module to be cloned + clone_parser.add_argument( + 'module', nargs=1, help='Name of the module to clone') + # Eventually specify where to clone the module + clone_parser.add_argument( + "clone_target", default=None, nargs="?", + help='Directory in which to clone the module') + clone_parser.set_defaults(command=self.clone) + + # Add an alias for historical reasons + co_parser = self.subparsers.add_parser( + 'co', parents=[clone_parser], conflict_handler='resolve', + help='Alias for clone') + co_parser.set_defaults(command=self.clone) + + def register_commit(self): + """Register the commit target and ci alias""" + + commit_parser = self.subparsers.add_parser( + 'commit', help='Commit changes', + description='This invokes a git commit. All tracked files with ' + 'changes will be committed unless a specific file ' + 'list is provided. $EDITOR will be used to generate a' + ' changelog message unless one is given to the ' + 'command. A push can be done at the same time.') + commit_parser.add_argument( + '-c', '--clog', default=False, action='store_true', + help='Generate the commit message from the Changelog section') + commit_parser.add_argument( + '--raw', action='store_true', default=False, + help='Make the clog raw') + commit_parser.add_argument( + '-t', '--tag', default=False, action='store_true', + help='Create a tag for this commit') + commit_parser.add_argument( + '-m', '--message', default=None, + help='Use the given as the commit message') + commit_parser.add_argument( + '-F', '--file', default=None, + help='Take the commit message from the given file') + # allow one to commit /and/ push at the same time. + commit_parser.add_argument( + '-p', '--push', default=False, action='store_true', + help='Commit and push as one action') + # Allow a list of files to be committed instead of everything + commit_parser.add_argument( + 'files', nargs='*', default=[], + help='Optional list of specific files to commit') + commit_parser.add_argument( + '-s', '--signoff', default=False, action='store_true', + help='Include a signed-off-by') + commit_parser.set_defaults(command=self.commit) + + # Add a ci alias + ci_parser = self.subparsers.add_parser( + 'ci', parents=[commit_parser], conflict_handler='resolve', + help='Alias for commit') + ci_parser.set_defaults(command=self.commit) + + def register_compile(self): + """Register the compile target""" + + compile_parser = self.subparsers.add_parser( + 'compile', parents=[self.rpm_parser_common], + help='Local test rpmbuild compile', + description='This command calls rpmbuild to compile the source. ' + 'By default the prep and configure stages will be ' + 'done as well, unless the short-circuit option is ' + 'used.') + compile_parser.add_argument('--short-circuit', + action='store_true', + help='short-circuit compile') + compile_parser.add_argument('--nocheck', + action='store_true', + help='nocheck compile') + compile_parser.set_defaults(command=self.compile) + + def register_diff(self): + """Register the diff target""" + + diff_parser = self.subparsers.add_parser( + 'diff', help='Show changes between commits, commit and working ' + 'tree, etc', + description='Use git diff to show changes that have been made to ' + 'tracked files. By default cached changes (changes ' + 'that have been git added) will not be shown.') + diff_parser.add_argument( + '--cached', default=False, action='store_true', + help='View staged changes') + diff_parser.add_argument( + 'files', nargs='*', default=[], + help='Optionally diff specific files') + diff_parser.set_defaults(command=self.diff) + + def register_gimmespec(self): + """Register the gimmespec target""" + + gimmespec_parser = self.subparsers.add_parser( + 'gimmespec', help='Print the spec file name') + gimmespec_parser.set_defaults(command=self.gimmespec) + + def register_gitbuildhash(self): + """Register the gitbuildhash target""" + + gitbuildhash_parser = self.subparsers.add_parser( + 'gitbuildhash', + help='Print the git hash used to build the provided n-v-r', + description='This will show you the commit hash string used to ' + 'build the provided build n-v-r') + gitbuildhash_parser.add_argument( + 'build', help='name-version-release of the build to query.') + gitbuildhash_parser.set_defaults(command=self.gitbuildhash) + + def register_giturl(self): + """Register the giturl target""" + + giturl_parser = self.subparsers.add_parser( + 'giturl', help='Print the git url for building', + description='This will show you which git URL would be used in a ' + 'build command. It uses the git hashsum of the HEAD ' + 'of the current branch (which may not be pushed).') + giturl_parser.set_defaults(command=self.giturl) + + def register_import_srpm(self): + """Register the import-srpm target""" + + import_srpm_parser = self.subparsers.add_parser( + 'import', help='Import srpm content into a module', + description='This will extract sources, patches, and the spec ' + 'file from an srpm and update the current module ' + 'accordingly. It will import to the current branch by ' + 'default.') + import_srpm_parser.add_argument( + '--skip-diffs', help="Don't show diffs when import srpms", + action='store_true') + import_srpm_parser.add_argument('srpm', help='Source rpm to import') + import_srpm_parser.set_defaults(command=self.import_srpm) + + def register_install(self): + """Register the install target""" + + install_parser = self.subparsers.add_parser( + 'install', parents=[self.rpm_parser_common], + help='Local test rpmbuild install', + description='This will call rpmbuild to run the install section. ' + 'All leading sections will be processed as well, ' + 'unless the short-circuit option is used.') + install_parser.add_argument( + '--short-circuit', + action='store_true', + default=False, + help='short-circuit install') + install_parser.add_argument( + '--nocheck', + action='store_true', + help='nocheck install') + install_parser.set_defaults(command=self.install, default=False) + + def register_lint(self): + """Register the lint target""" + + lint_parser = self.subparsers.add_parser( + 'lint', help='Run rpmlint against local spec and build output if ' + 'present.', + description='Rpmlint can be configured using the --rpmlintconf/-r' + ' option or by setting a .rpmlint file in the ' + 'working directory') + lint_parser.add_argument( + '--info', '-i', default=False, action='store_true', + help='Display explanations for reported messages') + lint_parser.add_argument( + '--rpmlintconf', '-r', default=None, + help='Use a specific configuration file for rpmlint') + lint_parser.set_defaults(command=self.lint) + + def register_local(self): + """Register the local target""" + + local_parser = self.subparsers.add_parser( + 'local', parents=[self.rpm_parser_common], + help='Local test rpmbuild binary', + description='Locally test run of rpmbuild producing binary RPMs. ' + 'The rpmbuild output will be logged into a file named' + ' .build-%{version}-%{release}.log') + # Allow the user to just pass "--md5" which will set md5 as the + # hash, otherwise use the default of sha256 + local_parser.add_argument( + '--md5', action='store_const', const='md5', default=None, + dest='hash', help='Use md5 checksums (for older rpm hosts)') + local_parser.set_defaults(command=self.local) + + def register_new(self): + """Register the new target""" + + new_parser = self.subparsers.add_parser( + 'new', help='Diff against last tag', + description='This will use git to show a diff of all the changes ' + '(even uncommitted changes) since the last git tag ' + 'was applied.') + new_parser.set_defaults(command=self.new) + + def register_mockbuild(self): + """Register the mockbuild target""" + + mockbuild_parser = self.subparsers.add_parser( + 'mockbuild', help='Local test build using mock', + description='This will use the mock utility to build the package ' + 'for the distribution detected from branch ' + 'information. This can be overridden using the global' + ' --dist option. Your user must be in the local ' + '"mock" group.', + epilog="If config file for mock isn't found in the " + "/etc/mock directory, a temporary config " + "directory for mock is created and populated " + "with a config file created with mock-config.") + mockbuild_parser.add_argument('--root', help='Override mock root') + # Allow the user to just pass "--md5" which will set md5 as the + # hash, otherwise use the default of sha256 + mockbuild_parser.add_argument( + '--md5', action='store_const', const='md5', default=None, + dest='hash', help='Use md5 checksums (for older rpm hosts)') + mockbuild_parser.add_argument( + '--no-clean', '-n', help='Do not clean chroot before building ' + 'package', action='store_true') + mockbuild_parser.add_argument( + '--no-cleanup-after', help='Do not clean chroot after building ' + '(if automatic cleanup is enabled', action='store_true') + mockbuild_parser.add_argument( + '--no-clean-all', '-N', help='Alias for both --no-clean and ' + '--no-cleanup-after', action='store_true') + mockbuild_parser.set_defaults(command=self.mockbuild) + + def register_mock_config(self): + """Register the mock-config target""" + + mock_config_parser = self.subparsers.add_parser( + 'mock-config', help='Generate a mock config', + description='This will generate a mock config based on the ' + 'buildsystem target') + mock_config_parser.add_argument( + '--target', help='Override target used for config', default=None) + mock_config_parser.add_argument('--arch', help='Override local arch') + mock_config_parser.set_defaults(command=self.mock_config) + + def register_new_sources(self): + """Register the new-sources target""" + + # Make it part of self to be used later + self.new_sources_parser = self.subparsers.add_parser( + 'new-sources', help='Upload new source files', + description='This will upload new source files to the lookaside ' + 'cache and remove any existing ones. The "sources" ' + 'and .gitignore files will be updated with the new ' + 'uploaded file(s).') + self.new_sources_parser.add_argument('files', nargs='+') + self.new_sources_parser.set_defaults( + command=self.new_sources, replace=True) + + def register_patch(self): + """Register the patch target""" + + patch_parser = self.subparsers.add_parser( + 'patch', help='Create and add a gendiff patch file', + epilog='Patch file will be named: package-version-suffix.patch ' + 'and the file will be added to the repo index') + patch_parser.add_argument( + '--rediff', action='store_true', default=False, + help='Recreate gendiff file retaining comments Saves old patch ' + 'file with a suffix of ~') + patch_parser.add_argument( + 'suffix', help='Look for files with this suffix to diff') + patch_parser.set_defaults(command=self.patch) + + def register_prep(self): + """Register the prep target""" + + prep_parser = self.subparsers.add_parser( + 'prep', parents=[self.rpm_parser_common], + help='Local test rpmbuild prep', + description='Use rpmbuild to "prep" the sources (unpack the ' + 'source archive(s) and apply any patches.)') + prep_parser.set_defaults(command=self.prep) + + def register_pull(self): + """Register the pull target""" + + pull_parser = self.subparsers.add_parser( + 'pull', help='Pull changes from the remote repository and update ' + 'the working copy.', + description='This command uses git to fetch remote changes and ' + 'apply them to the current working copy. A rebase ' + 'option is available which can be used to avoid ' + 'merges.', + epilog='See git pull --help for more details') + pull_parser.add_argument( + '--rebase', action='store_true', + help='Rebase the locally committed changes on top of the remote ' + 'changes after fetching. This can avoid a merge commit, but ' + 'does rewrite local history.') + pull_parser.add_argument( + '--no-rebase', action='store_true', + help='Do not rebase, overriding .git settings to the contrary') + pull_parser.set_defaults(command=self.pull) + + def register_push(self): + """Register the push target""" + + push_parser = self.subparsers.add_parser( + 'push', help='Push changes to remote repository') + push_parser.add_argument('--force', '-f', help='Force push', action='store_true') + push_parser.set_defaults(command=self.push) + + def register_scratch_build(self): + """Register the scratch-build target""" + + scratch_build_parser = self.subparsers.add_parser( + 'scratch-build', help='Request scratch build', + parents=[self.build_parser_common], + description='This command will request a scratch build of the ' + 'package. Without providing an srpm, it will attempt ' + 'to build the latest commit, which must have been ' + 'pushed. By default all appropriate arches will be ' + 'built.') + scratch_build_parser.add_argument( + '--srpm', nargs='?', const='CONSTRUCT', + help='Build from an srpm. If no srpm is provided with this ' + 'option an srpm will be generated from the current module ' + 'content.') + scratch_build_parser.set_defaults(command=self.scratch_build) + + def register_sources(self): + """Register the sources target""" + + sources_parser = self.subparsers.add_parser( + 'sources', help='Download source files') + sources_parser.add_argument( + '--outdir', default=os.curdir, + help='Directory to download files into (defaults to pwd)') + sources_parser.set_defaults(command=self.sources) + + def register_srpm(self): + """Register the srpm target""" + + srpm_parser = self.subparsers.add_parser( + 'srpm', help='Create a source rpm') + # optionally define old style hashsums + srpm_parser.add_argument( + '--md5', action='store_const', const='md5', default=None, + dest='hash', help='Use md5 checksums (for older rpm hosts)') + srpm_parser.set_defaults(command=self.srpm) + + def register_copr_build(self): + """Register the copr-build target""" + + copr_parser = self.subparsers.add_parser( + 'copr-build', help='Build package in Copr', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +Build package in Copr. + +Note: you need to have set up correct api key. For more information +see API KEY section of copr-cli(1) man page. +""") + + copr_parser.add_argument( + '--nowait', action='store_true', default=False, + help="Don't wait on build") + copr_parser.add_argument( + 'project', nargs=1, help='Name of the project in format USER/PROJECT') + copr_parser.set_defaults(command=self.copr_build) + + def register_switch_branch(self): + """Register the switch-branch target""" + + switch_branch_parser = self.subparsers.add_parser( + 'switch-branch', help='Work with branches', + description='This command can switch to a local git branch. If ' + 'provided with a remote branch name that does not ' + 'have a local match it will create one. It can also ' + 'be used to list the existing local and remote ' + 'branches.') + switch_branch_parser.add_argument( + 'branch', nargs='?', help='Branch name to switch to') + switch_branch_parser.add_argument( + '-l', '--list', action='store_true', + help='List both remote-tracking branches and local branches') + switch_branch_parser.add_argument( + '--fetch', help='Fetch new data from remote before switch', + action='store_true', dest='fetch') + switch_branch_parser.set_defaults(command=self.switch_branch) + + def register_tag(self): + """Register the tag target""" + + tag_parser = self.subparsers.add_parser( + 'tag', help='Management of git tags', + description='This command uses git to create, list, or delete ' + 'tags.') + tag_parser.add_argument( + '-f', '--force', default=False, + action='store_true', help='Force the creation of the tag') + tag_parser.add_argument( + '-m', '--message', default=None, + help='Use the given as the tag message') + tag_parser.add_argument( + '-c', '--clog', default=False, action='store_true', + help='Generate the tag message from the spec changelog section') + tag_parser.add_argument( + '--raw', action='store_true', default=False, + help='Make the clog raw') + tag_parser.add_argument( + '-F', '--file', default=None, + help='Take the tag message from the given file') + tag_parser.add_argument( + '-l', '--list', default=False, action='store_true', + help='List all tags with a given pattern, or all if not pattern ' + 'is given') + tag_parser.add_argument( + '-d', '--delete', default=False, action='store_true', + help='Delete a tag') + tag_parser.add_argument( + 'tag', nargs='?', default=None, help='Name of the tag') + tag_parser.set_defaults(command=self.tag) + + def register_unused_patches(self): + """Register the unused-patches target""" + + unused_patches_parser = self.subparsers.add_parser( + 'unused-patches', + help='Print list of patches not referenced by name in the ' + 'specfile') + unused_patches_parser.set_defaults(command=self.unused_patches) + + def register_upload(self): + """Register the upload target""" + + upload_parser = self.subparsers.add_parser( + 'upload', parents=[self.new_sources_parser], + conflict_handler='resolve', help='Upload source files', + description='This command will add a new source archive to the ' + 'lookaside cache. The sources and .gitignore file ' + 'will be updated with the new file(s).') + upload_parser.set_defaults(command=self.new_sources, replace=False) + + def register_verify_files(self): + """Register the verify-files target""" + + verify_files_parser = self.subparsers.add_parser( + 'verify-files', parents=[self.rpm_parser_common], + help='Locally verify %%files section', + description="Locally run 'rpmbuild -bl' to verify the spec file's" + " %files sections. This requires a successful run of " + "'{0} install' in advance.".format(self.name)) + verify_files_parser.set_defaults(command=self.verify_files) + + def register_verrel(self): + + verrel_parser = self.subparsers.add_parser( + 'verrel', help='Print the name-version-release') + verrel_parser.set_defaults(command=self.verrel) + + def register_container_build(self): + self.container_build_parser = \ + self.subparsers.add_parser('container-build', + help='build a container') + self.container_build_parser.add_argument('--repo-url', + metavar="URL", + help=("URL of yum repo file"), + nargs='*') + osbs_group = self.container_build_parser.add_argument_group('osbs') + osbs_group.add_argument('--osbs-config', + help="path to file with configuration of osbs", + metavar="PATH", + default=OSBS_DEFAULT_CONF_FILE) + osbs_group.add_argument('--instance', + help=("use specific instance specified " + "by section name in config"), + metavar="SECTION", default="default") + koji_group = self.container_build_parser.add_argument_group('koji') + koji_group.add_argument('--scratch', + help='Scratch build', + action="store_true") + + self.container_build_parser.add_argument( + '--target', + help='Override the default target', + default=None) + self.container_build_parser.add_argument( + '--build-with', + help='Build container with specified builder type. Default is koji', + dest="build_with", + choices=("koji", "osbs"), + default="koji") + self.container_build_parser.add_argument( + '--nowait', + action='store_true', + default=False, + help="Don't wait on build") + + self.container_build_parser.set_defaults(command=self.container_build) + + def register_container_build_setup(self): + self.container_build_setup_parser = \ + self.subparsers.add_parser('container-build-setup', + help='set options for container-build') + group = self.container_build_setup_parser.add_mutually_exclusive_group(required=True) + group.add_argument( + '--get-autorebuild', + help='Get autorebuild value', + action='store_true', + default=None) + group.add_argument( + '--set-autorebuild', + help='Turn autorebuilds on/off', + choices=('true', 'false'), + default=None) + self.container_build_setup_parser.set_defaults( + command=self.container_build_setup) + + # All the command functions go here + def usage(self): + self.parser.print_help() + + def build(self, sets=None): + # We may have gotten arches by way of scratch build, so handle them + arches = None + if hasattr(self.args, 'arches'): + arches = self.args.arches + # Place holder for if we build with an uploaded srpm or not + url = None + # See if this is a chain or not + chain = None + if hasattr(self.args, 'chain'): + chain = self.args.chain + # Need to do something with BUILD_FLAGS or KOJI_FLAGS here for compat + if self.args.target: + self.cmd._target = self.args.target + # handle uploading the srpm if we got one + if hasattr(self.args, 'srpm') and self.args.srpm: + # See if we need to generate the srpm first + if self.args.srpm == 'CONSTRUCT': + self.log.debug('Generating an srpm') + self.srpm() + self.args.srpm = '%s.src.rpm' % self.cmd.nvr + # Figure out if we want a verbose output or not + callback = None + if not self.args.q: + callback = self._progress_callback + # define a unique path for this upload. Stolen from /usr/bin/koji + uniquepath = ('cli-build/%r.%s' + % (time.time(), + ''.join([random.choice(string.ascii_letters) + for i in range(8)]))) + # Should have a try here, not sure what errors we'll get yet though + self.cmd.koji_upload(self.args.srpm, uniquepath, callback=callback) + if not self.args.q: + # print an extra blank line due to callback oddity + print('') + url = '%s/%s' % (uniquepath, os.path.basename(self.args.srpm)) + # nvr_check option isn't set by all commands which calls this + # function so handle it as an optional argument + nvr_check = True + if hasattr(self.args, 'nvr_check'): + nvr_check = self.args.nvr_check + task_id = self.cmd.build(self.args.skip_tag, self.args.scratch, + self.args.background, url, chain, arches, + sets, nvr_check) + + # Log out of the koji session + self.cmd.kojisession.logout() + + if self.args.nowait: + return + + # Pass info off to our koji task watcher + return self._watch_koji_tasks(self.cmd.kojisession, [task_id]) + + def chainbuild(self): + if self.cmd.module_name in self.args.package: + raise Exception('%s must not be in the chain' % self.cmd.module_name) + + # make sure we didn't get an empty chain + if self.args.package == [':']: + raise Exception('Must provide at least one dependency build') + + # Break the chain up into sections + sets = False + urls = [] + build_set = [] + self.log.debug('Processing chain %s' % ' '.join(self.args.package)) + for component in self.args.package: + if component == ':': + # We've hit the end of a set, add the set as a unit to the + # url list and reset the build_set. + urls.append(build_set) + self.log.debug('Created a build set: %s', ' '.join(build_set)) + build_set = [] + sets = True + else: + # Figure out the scm url to build from package name + hash = self.cmd.get_latest_commit(component, self.cmd.branch_merge) + url = self.cmd.anongiturl % {'module': component} + '#%s' % hash + # If there are no ':' in the chain list, treat each object as + # an individual chain + if ':' in self.args.package: + build_set.append(url) + else: + urls.append([url]) + self.log.debug('Created a build set: %s', url) + # Take care of the last build set if we have one + if build_set: + self.log.debug('Created a build set: %s', ' '.join(build_set)) + urls.append(build_set) + # See if we ended in a : making our last build it's own group + if self.args.package[-1] == ':': + self.log.debug('Making the last build its own set.') + urls.append([]) + # pass it off to build + self.args.chain = urls + self.args.skip_tag = False + self.args.scratch = False + return self.build(sets=sets) + + def clean(self): + dry = False + useignore = True + if self.args.dry_run: + dry = True + if self.args.x: + useignore = False + return self.cmd.clean(dry, useignore) + + def clog(self): + self.cmd.clog(raw=self.args.raw) + + def clone(self): + if self.args.branches: + self.cmd.clone_with_dirs(self.args.module[0], + anon=self.args.anonymous, + target=self.args.clone_target) + else: + self.cmd.clone(self.args.module[0], + branch=self.args.branch, + anon=self.args.anonymous, + target=self.args.clone_target) + + def commit(self): + if self.args.clog: + self.cmd.clog(self.args.raw) + self.args.file = os.path.abspath(os.path.join(self.args.path, + 'clog')) + try: + self.cmd.commit(self.args.message, self.args.file, + self.args.files, self.args.signoff) + except Exception: + if self.args.tag: + self.log.error('Could not commit, will not tag!') + if self.args.push: + self.log.error('Could not commit, will not push!') + raise + + try: + if self.args.tag: + tagname = self.cmd.nvr + self.cmd.add_tag(tagname, True, self.args.message, + self.args.file) + except Exception: + if self.args.push: + self.log.error('Could not tag, will not push!') + raise + + if self.args.push: + self.push() + + def compile(self): + arch = None + short = False + nocheck = False + if self.args.arch: + arch = self.args.arch + if self.args.short_circuit: + short = True + if self.args.nocheck: + nocheck = True + self.cmd.compile(arch=arch, short=short, + builddir=self.args.builddir, nocheck=nocheck) + + def container_build(self): + if self.args.build_with == "koji": + self.container_build_koji() + elif self.args.build_with == "osbs": + self.container_build_osbs() + + def container_build_koji(self): + target_override = False + # Override the target if we were supplied one + if self.args.target: + self.cmd._target = self.args.target + target_override = True + + opts = {"scratch": self.args.scratch, + "quiet": self.args.q, + "yum_repourls": self.args.repo_url, + "git_branch": self.cmd.branch_merge} + + section_name = "%s.container-build" % self.name + err_msg = "Missing {option} option in [{plugin.section}] section. "\ + "Using {option} from [{root.section}]" + err_args = {"plugin.section": section_name, "root.section": self.name} + + if self.config.has_option(section_name, "kojiconfig"): + kojiconfig = self.config.get(section_name, "kojiconfig") + else: + err_args["option"] = "kojiconfig" + self.log.debug(err_msg % err_args) + kojiconfig = self.config.get(self.name, "kojiconfig") + + if self.config.has_option(section_name, "build_client"): + build_client = self.config.get(section_name, "build_client") + else: + err_args["option"] = "kojiconfig" + self.log.debug(err_msg % err_args) + build_client = self.config.get(self.name, "build_client") + + self.cmd.container_build_koji(target_override, opts=opts, + kojiconfig=kojiconfig, + build_client=build_client, + koji_task_watcher=self._watch_koji_tasks, + nowait=self.args.nowait) + + def container_build_osbs(self): + target_override = False + # Override the target if we were supplied one + if self.args.target: + self.cmd._target = self.args.target + target_override = True + + self.cmd.osbs_build( + config_file=self.args.osbs_config, + config_section=self.args.instance, + target_override=target_override, + yum_repourls=self.args.repo_url, + nowait=self.args.nowait + ) + + def container_build_setup(self): + self.cmd.container_build_setup(get_autorebuild=self.args.get_autorebuild, + set_autorebuild=self.args.set_autorebuild) + + def copr_build(self): + self.log.debug('Generating an srpm') + self.args.hash = None + self.srpm() + srpm_name = '%s.src.rpm' % self.cmd.nvr + self.cmd.copr_build(self.args.project[0], srpm_name, self.args.nowait) + + def diff(self): + self.cmd.diff(self.args.cached, self.args.files) + + def gimmespec(self): + print(self.cmd.spec) + + def gitbuildhash(self): + print(self.cmd.gitbuildhash(self.args.build)) + + def giturl(self): + print(self.cmd.giturl()) + + def import_srpm(self): + uploadfiles = self.cmd.import_srpm(self.args.srpm) + if uploadfiles: + self.cmd.upload(uploadfiles, replace=True) + if not self.args.skip_diffs: + self.cmd.diff(cached=True) + self.log.info('--------------------------------------------') + self.log.info("New content staged and new sources uploaded.") + self.log.info("Commit if happy or revert with: git reset --hard HEAD") + + def install(self): + self.cmd.install(arch=self.args.arch, + short=self.args.short_circuit, + builddir=self.args.builddir, + nocheck=self.args.nocheck) + + def lint(self): + self.cmd.lint(self.args.info, self.args.rpmlintconf) + + def local(self): + self.cmd.local(arch=self.args.arch, hashtype=self.args.hash, + builddir=self.args.builddir) + + def mockbuild(self): + try: + self.cmd.sources() + except Exception as e: + self.log.error('Could not download sources: %s' % e) + sys.exit(1) + + mockargs = [] + + if self.args.no_clean or self.args.no_clean_all: + mockargs.append('--no-clean') + + if self.args.no_cleanup_after or self.args.no_clean_all: + mockargs.append('--no-cleanup-after') + + # Pick up any mockargs from the env + try: + mockargs += os.environ['MOCKARGS'].split() + except KeyError: + # there were no args + pass + try: + self.cmd.mockbuild(mockargs, self.args.root, + hashtype=self.args.hash) + except Exception as e: + self.log.error('Could not run mockbuild: %s' % e) + sys.exit(1) + + def mock_config(self): + try: + print(self.cmd.mock_config(self.args.target, self.args.arch)) + except Exception as e: + self.log.error('Could not generate the mock config: %s' % e) + sys.exit(1) + + def new(self): + print(self.cmd.new()) + + def new_sources(self): + # Check to see if the files passed exist + for file in self.args.files: + if not os.path.isfile(file): + raise Exception('Path does not exist or is ' + 'not a file: %s' % file) + self.cmd.upload(self.args.files, replace=self.args.replace) + self.log.info("Source upload succeeded. Don't forget to commit the " + "sources file") + + def patch(self): + self.cmd.patch(self.args.suffix, rediff=self.args.rediff) + + def prep(self): + self.cmd.prep(arch=self.args.arch, builddir=self.args.builddir) + + def pull(self): + self.cmd.pull(rebase=self.args.rebase, + norebase=self.args.no_rebase) + + def push(self): + self.cmd.push(getattr(self.args, 'force', False)) + + def scratch_build(self): + # A scratch build is just a build with --scratch + self.args.scratch = True + self.args.skip_tag = False + return self.build() + + def sources(self): + self.cmd.sources(self.args.outdir) + + def srpm(self): + self.cmd.sources() + self.cmd.srpm(hashtype=self.args.hash) + + def switch_branch(self): + if self.args.branch: + self.cmd.switch_branch(self.args.branch, self.args.fetch) + else: + (locals, remotes) = self.cmd._list_branches(self.args.fetch) + # This is some ugly stuff here, but trying to emulate + # the way git branch looks + locals = [' %s ' % branch for branch in locals] + local_branch = self.cmd.repo.active_branch.name + locals[locals.index(' %s ' % + local_branch)] = '* %s' % local_branch + print('Locals:\n%s\nRemotes:\n %s' % + ('\n'.join(locals), '\n '.join(remotes))) + + def tag(self): + if self.args.list: + self.cmd.list_tag(self.args.tag) + elif self.args.delete: + self.cmd.delete_tag(self.args.tag) + else: + filename = self.args.file + tagname = self.args.tag + if not tagname or self.args.clog: + if not tagname: + tagname = self.cmd.nvr + if self.args.clog: + self.cmd.clog(self.args.raw) + filename = 'clog' + self.cmd.add_tag(tagname, self.args.force, + self.args.message, filename) + + def unused_patches(self): + unused = self.cmd.unused_patches() + print('\n'.join(unused)) + + def verify_files(self): + self.cmd.verify_files(builddir=self.args.builddir) + + def verrel(self): + print('%s-%s-%s' % (self.cmd.module_name, self.cmd.ver, + self.cmd.rel)) + + # Other class stuff goes here + # The next 6 functions come from the koji project, from /usr/bin/koji + # They should be in a library somewhere, but I have to steal them. + # The code is licensed LGPLv2.1 and thus my (slightly) derived code + # is as well. + def _display_tasklist_status(self, tasks): + free = 0 + open = 0 + failed = 0 + done = 0 + for task_id in tasks.keys(): + status = tasks[task_id].info['state'] + if status == koji.TASK_STATES['FAILED']: + failed += 1 + elif status in (koji.TASK_STATES['CLOSED'], + koji.TASK_STATES['CANCELED']): + done += 1 + elif status in (koji.TASK_STATES['OPEN'], + koji.TASK_STATES['ASSIGNED']): + open += 1 + elif status == koji.TASK_STATES['FREE']: + free += 1 + self.log.info(" %d free %d open %d done %d failed" % + (free, open, done, failed)) + + def _display_task_results(self, tasks): + for task in [task for task in tasks.values() if task.level == 0]: + state = task.info['state'] + task_label = task.str() + + if state == koji.TASK_STATES['CLOSED']: + self.log.info('%s completed successfully' % task_label) + elif state == koji.TASK_STATES['FAILED']: + self.log.info('%s failed' % task_label) + elif state == koji.TASK_STATES['CANCELED']: + self.log.info('%s was canceled' % task_label) + else: + # shouldn't happen + self.log.info('%s has not completed' % task_label) + + def _watch_koji_tasks(self, session, tasklist): + if not tasklist: + return + self.log.info('Watching tasks (this may be safely interrupted)...') + # Place holder for return value + rv = 0 + try: + tasks = {} + for task_id in tasklist: + tasks[task_id] = TaskWatcher(task_id, session, self.log, + quiet=self.args.q) + while True: + all_done = True + for task_id, task in tasks.items(): + changed = task.update() + if not task.is_done(): + all_done = False + else: + if changed: + # task is done and state just changed + if not self.args.q: + self._display_tasklist_status(tasks) + if not task.is_success(): + rv = 1 + for child in session.getTaskChildren(task_id): + child_id = child['id'] + if child_id not in tasks.keys(): + tasks[child_id] = TaskWatcher(child_id, + session, + self.log, + task.level + 1, + quiet=self.args.q) + tasks[child_id].update() + # If we found new children, go through the list + # again, in case they have children also + all_done = False + if all_done: + if not self.args.q: + print("") + self._display_task_results(tasks) + break + + time.sleep(1) + except (KeyboardInterrupt): + if tasks: + self.log.info(""" +Tasks still running. You can continue to watch with the '%s watch-task' command. + Running Tasks: + %s""" + % (self.config.get(self.name, 'build_client'), + '\n'.join(['%s: %s' % (t.str(), + t.display_state(t.info)) + for t in tasks.values() + if not t.is_done()]))) + # A ^c should return non-zero so that it doesn't continue + # on to any && commands. + rv = 1 + return rv + + # Stole these three functions from /usr/bin/koji + def _format_size(self, size): + if (size / 1073741824 >= 1): + return "%0.2f GiB" % (size / 1073741824.0) + if (size / 1048576 >= 1): + return "%0.2f MiB" % (size / 1048576.0) + if (size / 1024 >= 1): + return "%0.2f KiB" % (size / 1024.0) + return "%0.2f B" % (size) + + def _format_secs(self, t): + h = t / 3600 + t = t % 3600 + m = t / 60 + s = t % 60 + return "%02d:%02d:%02d" % (h, m, s) + + def _progress_callback(self, uploaded, total, piece, time, total_time): + percent_done = float(uploaded)/float(total) + percent_done_str = "%02d%%" % (percent_done * 100) + data_done = self._format_size(uploaded) + elapsed = self._format_secs(total_time) + + speed = "- B/sec" + if (time): + if (uploaded != total): + speed = self._format_size(float(piece)/float(time)) + "/sec" + else: + speed = self._format_size(float(total)/float(total_time)) + \ + "/sec" + + # write formatted string and flush + sys.stdout.write("[% -36s] % 4s % 8s % 10s % 14s\r" % + ('='*(int(percent_done*36)), + percent_done_str, elapsed, data_done, speed)) + sys.stdout.flush() + + def setupLogging(self, log): + """Setup the various logging stuff.""" + + # Assign the log object to self + self.log = log + + # Add a log filter class + class StdoutFilter(logging.Filter): + + def filter(self, record): + # If the record level is 20 (INFO) or lower, let it through + return record.levelno <= logging.INFO + + # have to create a filter for the stdout stream to filter out WARN+ + myfilt = StdoutFilter() + # Simple format + formatter = logging.Formatter('%(message)s') + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.addFilter(myfilt) + stdouthandler.setFormatter(formatter) + stderrhandler = logging.StreamHandler() + stderrhandler.setLevel(logging.WARNING) + stderrhandler.setFormatter(formatter) + self.log.addHandler(stdouthandler) + self.log.addHandler(stderrhandler) + + def parse_cmdline(self, manpage=False): + """Parse the commandline, optionally make a manpage + + This also sets up self.user + """ + + if manpage: + # Generate the man page + man_name = self.name + if man_name.endswith('.py'): + man_name = man_name[:-3] + man_page = __import__('%s' % man_name) + man_page.generate(self.parser, self.subparsers) + sys.exit(0) + # no return possible + + # Parse the args + self.args = self.parser.parse_args() + if self.args.user: + self.user = self.args.user + else: + self.user = pwd.getpwuid(os.getuid())[0] + + +# Add a class stolen from /usr/bin/koji to watch tasks +# this was cut/pasted from koji, and then modified for local use. +# The formatting is koji style, not the stile of this file. Do not use these +# functions as a style guide. +# This is fragile and hopefully will be replaced by a real kojiclient lib. + + +class TaskWatcher(object): + + def __init__(self, task_id, session, log, level=0, quiet=False): + self.id = task_id + self.session = session + self.info = None + self.level = level + self.quiet = quiet + self.log = log + + # XXX - a bunch of this stuff needs to adapt to different tasks + + def str(self): + if self.info: + label = koji.taskLabel(self.info) + return "%s%d %s" % (' ' * self.level, self.id, label) + else: + return "%s%d" % (' ' * self.level, self.id) + + def __str__(self): + return self.str() + + def get_failure(self): + """Print information about task completion""" + if self.info['state'] != koji.TASK_STATES['FAILED']: + return '' + error = None + try: + self.session.getTaskResult(self.id) + except (xmlrpc_client.Fault, koji.GenericError) as e: + error = e + if error is None: + # print "%s: complete" % self.str() + # We already reported this task as complete in update() + return '' + else: + return '%s: %s' % (error.__class__.__name__, str(error).strip()) + + def update(self): + """Update info and log if needed. Returns True on state change.""" + if self.is_done(): + # Already done, nothing else to report + return False + last = self.info + try: + self.info = self.session.getTaskInfo(self.id, request=True) + except koji.GenericError: + raise Exception("No such task id: %i" % self.id) + state = self.info['state'] + if last: + # compare and note status changes + laststate = last['state'] + if laststate != state: + self.log.info("%s: %s -> %s", + self.str(), self.display_state(last), self.display_state(self.info)) + return True + return False + else: + # First time we're seeing this task, so just show the current state + self.log.info("%s: %s", self.str(), self.display_state(self.info)) + return False + + def is_done(self): + if self.info is None: + return False + state = koji.TASK_STATES[self.info['state']] + return (state in ['CLOSED', 'CANCELED', 'FAILED']) + + def is_success(self): + if self.info is None: + return False + state = koji.TASK_STATES[self.info['state']] + return (state == 'CLOSED') + + def display_state(self, info): + # We can sometimes be passed a task that is not yet open, but + # not finished either. info would be none. + if not info: + return 'unknown' + if info['state'] == koji.TASK_STATES['OPEN']: + if info['host_id']: + host = self.session.getHost(info['host_id']) + return 'open (%s)' % host['name'] + else: + return 'open' + elif info['state'] == koji.TASK_STATES['FAILED']: + return 'FAILED: %s' % self.get_failure() + else: + return koji.TASK_STATES[info['state']].lower() + + +if __name__ == '__main__': + client = cliClient() + client.do_imports() + client.parse_cmdline() + + if not client.args.path: + try: + client.args.path = os.getcwd() + except: + print('Could not get current path, have you deleted it?') + sys.exit(1) + + # setup the logger -- This logger will take things of INFO or DEBUG and + # log it to stdout. Anything above that (WARN, ERROR, CRITICAL) will go + # to stderr. Normal operation will show anything INFO and above. + # Quiet hides INFO, while Verbose exposes DEBUG. In all cases WARN or + # higher are exposed (via stderr). + log = client.site.log + client.setupLogging(log) + + if client.args.v: + log.setLevel(logging.DEBUG) + elif client.args.q: + log.setLevel(logging.WARNING) + else: + log.setLevel(logging.INFO) + + # Run the necessary command + try: + client.args.command() + except KeyboardInterrupt: + pass diff --git a/pyrpkg/errors.py b/pyrpkg/errors.py new file mode 100644 index 0000000..5ea5f08 --- /dev/null +++ b/pyrpkg/errors.py @@ -0,0 +1,53 @@ +# Copyright (c) 2015 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +"""Custom error classes""" + + +class rpkgError(Exception): + """Our base error class""" + faultCode = 1000 + + +class rpkgAuthError(rpkgError): + """Raised in case of authentication errors""" + faultCode = 1002 + + +class UnknownTargetError(Exception): + faultCode = 1004 + + +class HashtypeMixingError(rpkgError): + """Raised when we try to mix hash types in a sources file""" + def __init__(self, existing_hashtype, new_hashtype): + super(HashtypeMixingError, self).__init__() + + self.existing_hashtype = existing_hashtype + self.new_hashtype = new_hashtype + + +class MalformedLineError(rpkgError): + """Raised when parsing a sources file with malformed lines""" + pass + + +class InvalidHashType(rpkgError): + """Raised when we don't know the requested hash algorithm""" + pass + + +class DownloadError(rpkgError): + """Raised when something went wrong during a download""" + pass + + +class UploadError(rpkgError): + """Raised when something went wrong during an upload""" + pass diff --git a/pyrpkg/gitignore.py b/pyrpkg/gitignore.py new file mode 100644 index 0000000..db3f3a5 --- /dev/null +++ b/pyrpkg/gitignore.py @@ -0,0 +1,90 @@ +# Copyright (c) 2015 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +"""Manage a .gitignore file""" + + +import fnmatch +import os + + +class GitIgnore(object): + """A class to manage a .gitignore file""" + def __init__(self, path): + """Constructor + + Args: + path (str): The full path to the .gitignore file. If it does not + exist, the file will be created when running GitIgnore.write() + for the first time. + """ + self.path = path + + # Lines of the .gitignore file, used to check if entries need to be + # added or already exist. + self.__lines = [] + + if os.path.exists(self.path): + with open(self.path, 'r') as f: + for line in f: + self.__lines.append(self.__ensure_newline(line)) + + # Set to True if we end up making any modifications, used to + # prevent unnecessary writes. + self.modified = False + + def __ensure_newline(self, line): + return line if line.endswith('\n') else '%s\n' % line + + def add(self, line): + """Add a line + + Args: + line (str): The line to add to the file. It will not be added if + it already matches an existing line. + """ + if self.match(line): + return + + line = self.__ensure_newline(line) + self.__lines.append(line) + self.modified = True + + def match(self, line): + """Check whether the line matches an existing one + + This uses fnmatch to match against wildcards. + + Args: + line (str): The new line to match against existing ones. + + Returns: + True if the new line matches, False otherwise. + """ + line = line.lstrip('/').rstrip('\n') + + for entry in self.__lines: + entry = entry.lstrip('/').rstrip('\n') + if fnmatch.fnmatch(line, entry): + return True + + return False + + def write(self): + """Write the file to the disk + + This will only actually write if necessary, that is if lines have been + added since the last time the file was written. + """ + if self.modified: + with open(self.path, 'w') as f: + for line in self.__lines: + f.write(line) + + self.modified = False diff --git a/pyrpkg/lookaside.py b/pyrpkg/lookaside.py new file mode 100644 index 0000000..e257208 --- /dev/null +++ b/pyrpkg/lookaside.py @@ -0,0 +1,307 @@ +# Copyright (c) 2015 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +"""Interact with a lookaside cache + +This module contains everything needed to upload and download source files the +way it is done by Fedora, RHEL, and other distributions maintainers. +""" + + +import hashlib +import io +import logging +import os +import sys + +import pycurl + +from .errors import DownloadError, InvalidHashType, UploadError + + +class CGILookasideCache(object): + """A class to interact with a CGI-based lookaside cache""" + def __init__(self, hashtype, download_url, upload_url, + client_cert=None, ca_cert=None): + """Constructor + + Args: + hashtype (str): The hash algorithm to use for uploads. (e.g 'md5') + download_url (str): The URL used to download source files. + upload_url (str): The URL of the CGI script called when uploading + source files. + client_cert (str, optional): The full path to the client-side + certificate to use for HTTPS authentication. It defaults to + None, in which case no client-side certificate is used. + ca_cert (str, optional): The full path to the CA certificate to + use for HTTPS connexions. (e.g if the server certificate is + self-signed. It defaults to None, in which case the system CA + bundle is used. + """ + self.hashtype = hashtype + self.download_url = download_url + self.upload_url = upload_url + self.client_cert = client_cert + self.ca_cert = ca_cert + + self.log = logging.getLogger(__name__) + + self.download_path = '%(name)s/%(filename)s/%(hash)s/%(filename)s' + + def print_progress(self, to_download, downloaded, to_upload, uploaded): + if not sys.stdout.isatty(): + # Don't print progress if not outputting into TTY. The progress + # output is not useful in logs. + return + + if to_download > 0: + done = downloaded / to_download + + elif to_upload > 0: + done = uploaded / to_upload + + else: + return + + done_chars = int(done * 72) + remain_chars = 72 - done_chars + done = int(done * 1000) / 10.0 + + p = "\r%s%s %s%%" % ("#" * done_chars, " " * remain_chars, done) + sys.stdout.write(p) + sys.stdout.flush() + + def hash_file(self, filename, hashtype=None): + """Compute the hash of a file + + Args: + filename (str): The full path to the file. It is assumed to exist. + hashtype (str, optional): The hash algorithm to use. (e.g 'md5') + This defaults to the hashtype passed to the constructor. + + Returns: + The hash digest. + """ + if hashtype is None: + hashtype = self.hashtype + + try: + sum = hashlib.new(hashtype) + + except ValueError: + raise InvalidHashType(hashtype) + + with open(filename, 'rb') as f: + chunk = f.read(8192) + + while chunk: + sum.update(chunk) + chunk = f.read(8192) + + return sum.hexdigest() + + def file_is_valid(self, filename, hash, hashtype=None): + """Ensure the file is correct + + Args: + filename (str): The full path to the file. It is assumed to exist. + hash (str): The known good hash of the file. + hashtype (str, optional): The hash algorithm to use. (e.g 'md5') + This defaults to the hashtype passed to the constructor. + + Returns: + True if the file is valid, False otherwise. + """ + sum = self.hash_file(filename, hashtype) + return sum == hash + + def download(self, name, filename, hash, outfile, hashtype=None, **kwargs): + """Download a source file + + Args: + name (str): The name of the module. (usually the name of the SRPM) + filename (str): The name of the file to download. + hash (str): The known good hash of the file. + outfile (str): The full path where to save the downloaded file. + hashtype (str, optional): The hash algorithm. (e.g 'md5') + This defaults to the hashtype passed to the constructor. + **kwargs: Additional keyword arguments. They will be used when + contructing the full URL to the file to download. + """ + if hashtype is None: + hashtype = self.hashtype + + if os.path.exists(outfile): + if self.file_is_valid(outfile, hash, hashtype=hashtype): + return + + self.log.info("Downloading %s", filename) + urled_file = filename.replace(' ', '%20') + + path_dict = {'name': name, 'filename': urled_file, 'hash': hash, + 'hashtype': hashtype} + path_dict.update(kwargs) + path = self.download_path % path_dict + url = '%s/%s' % (self.download_url, path) + self.log.debug("Full url: %s" % url) + + with open(outfile, 'wb') as f: + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.HTTPHEADER, ['Pragma:']) + c.setopt(pycurl.NOPROGRESS, False) + c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress) + c.setopt(pycurl.OPT_FILETIME, True) + c.setopt(pycurl.WRITEDATA, f) + + try: + c.perform() + tstamp = c.getinfo(pycurl.INFO_FILETIME) + status = c.getinfo(pycurl.RESPONSE_CODE) + + except Exception as e: + raise DownloadError(e) + + finally: + c.close() + + # Get back a new line, after displaying the download progress + sys.stdout.write('\n') + sys.stdout.flush() + + if status != 200: + self.log.info('Remove downloaded invalid file %s', outfile) + os.remove(outfile) + raise DownloadError('Server returned status code %d' % status) + + os.utime(outfile, (tstamp, tstamp)) + + if not self.file_is_valid(outfile, hash, hashtype=hashtype): + raise DownloadError('%s failed checksum' % filename) + + def remote_file_exists(self, name, filename, hash): + """Verify whether a file exists on the lookaside cache + + Args: + name: The name of the module. (usually the name of the SRPM) + filename: The name of the file to check for. + hash: The known good hash of the file. + """ + post_data = [('name', name), + ('%ssum' % self.hashtype, hash), + ('filename', filename)] + + with io.BytesIO() as buf: + c = pycurl.Curl() + c.setopt(pycurl.URL, self.upload_url) + c.setopt(pycurl.WRITEFUNCTION, buf.write) + c.setopt(pycurl.HTTPPOST, post_data) + + if self.client_cert is not None: + if os.path.exists(self.client_cert): + c.setopt(pycurl.SSLCERT, self.client_cert) + else: + self.log.warning("Missing certificate: %s" + % self.client_cert) + + if self.ca_cert is not None: + if os.path.exists(self.ca_cert): + c.setopt(pycurl.CAINFO, self.ca_cert) + else: + self.log.warning("Missing certificate: %s" % self.ca_cert) + + try: + c.perform() + status = c.getinfo(pycurl.RESPONSE_CODE) + + except Exception as e: + raise UploadError(e) + + finally: + c.close() + + output = buf.getvalue().strip() + + if status != 200: + raise UploadError(output) + + # Lookaside CGI script returns these strings depending on whether + # or not the file exists: + if output == b'Available': + return True + + if output == b'Missing': + return False + + # Something unexpected happened + self.log.debug(output) + raise UploadError('Error checking for %s at %s' + % (filename, self.upload_url)) + + def upload(self, name, filepath, hash): + """Upload a source file + + Args: + name (str): The name of the module. (usually the name of the SRPM) + filepath (str): The full path to the file to upload. + hash (str): The known good hash of the file. + """ + filename = os.path.basename(filepath) + + if self.remote_file_exists(name, filename, hash): + self.log.info("File already uploaded: %s" % filepath) + return + + self.log.info("Uploading: %s" % filepath) + post_data = [('name', name), + ('%ssum' % self.hashtype, hash), + ('file', (pycurl.FORM_FILE, filepath))] + + with io.BytesIO() as buf: + c = pycurl.Curl() + c.setopt(pycurl.URL, self.upload_url) + c.setopt(pycurl.NOPROGRESS, False) + c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress) + c.setopt(pycurl.WRITEFUNCTION, buf.write) + c.setopt(pycurl.HTTPPOST, post_data) + + if self.client_cert is not None: + if os.path.exists(self.client_cert): + c.setopt(pycurl.SSLCERT, self.client_cert) + else: + self.log.warning("Missing certificate: %s" + % self.client_cert) + + if self.ca_cert is not None: + if os.path.exists(self.ca_cert): + c.setopt(pycurl.CAINFO, self.ca_cert) + else: + self.log.warning("Missing certificate: %s" % self.ca_cert) + + try: + c.perform() + status = c.getinfo(pycurl.RESPONSE_CODE) + + except Exception as e: + raise UploadError(e) + + finally: + c.close() + + output = buf.getvalue().strip() + + # Get back a new line, after displaying the download progress + sys.stdout.write('\n') + sys.stdout.flush() + + if status != 200: + raise UploadError(output) + + if output: + self.log.debug(output) diff --git a/pyrpkg/sources.py b/pyrpkg/sources.py new file mode 100644 index 0000000..a443f38 --- /dev/null +++ b/pyrpkg/sources.py @@ -0,0 +1,108 @@ +""" +Our so-called sources file is simple text-based line-oriented file format. + +Each line represents one source file and is in the same format as the output +of commands like `md5sum --tag filename`: + + hashtype (filename) = hash + +To preserve backwards compatibility, lines can also be in the older format, +which corresponds to the output of commands like `md5sum filename`: + + hash filename + +This module implements a simple API to read these files, parse lines into +entries, and write these entries to the file in the proper format. +""" + + +import os +import re + +from .errors import HashtypeMixingError, MalformedLineError + + +LINE_PATTERN = re.compile( + r'^(?P[^ ]+?) \((?P[^ )]+?)\) = (?P[^ ]+?)$') + + +class SourcesFile(object): + def __init__(self, sourcesfile, entry_type, replace=False): + self.sourcesfile = sourcesfile + self.entry_type = {'old': SourceFileEntry, + 'bsd': BSDSourceFileEntry}[entry_type] + self.entries = [] + + if not replace: + if not os.path.exists(sourcesfile): + return + + with open(sourcesfile) as f: + for line in f: + entry = self.parse_line(line) + + if entry and entry not in self.entries: + self.entries.append(entry) + + def __contains__(self, filename): + for entry in self.entries: + if entry.file == filename: + return True + return False + + def parse_line(self, line): + stripped = line.strip() + + if not stripped: + return + + m = LINE_PATTERN.match(stripped) + if m is not None: + return self.entry_type(m.group('hashtype'), m.group('file'), + m.group('hash')) + + # Try falling back on the old format + try: + hash, file = stripped.split(' ', 1) + + except ValueError: + raise MalformedLineError(line) + + return self.entry_type('md5', file, hash) + + def add_entry(self, hashtype, file, hash): + entry = self.entry_type(hashtype, file, hash) + + for e in self.entries: + if entry.hashtype != e.hashtype: + raise HashtypeMixingError(e.hashtype, entry.hashtype) + + if entry == e: + return + + self.entries.append(entry) + + def write(self): + with open(self.sourcesfile, 'w') as f: + for entry in self.entries: + f.write(str(entry)) + + +class SourceFileEntry(object): + def __init__(self, hashtype, file, hash): + self.hashtype = hashtype.lower() + self.hash = hash + self.file = file + + def __str__(self): + return '%s %s\n' % (self.hash, self.file) + + def __eq__(self, other): + return ((self.hashtype, self.hash, self.file) == + (other.hashtype, other.hash, other.file)) + + +class BSDSourceFileEntry(SourceFileEntry): + def __str__(self): + return '%s (%s) = %s\n' % (self.hashtype.upper(), self.file, + self.hash) diff --git a/pyrpkg/utils.py b/pyrpkg/utils.py new file mode 100644 index 0000000..03606a4 --- /dev/null +++ b/pyrpkg/utils.py @@ -0,0 +1,95 @@ +# Copyright (c) 2015 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +"""Miscellaneous utilities + +This module contains a bunch of utilities used elsewhere in pyrpkg. +""" + + +import warnings + +import os +import six + +if six.PY3: + def u(s): + return s + + getcwd = os.getcwd +else: + def u(s): + return s.decode('utf-8') + + getcwd = os.getcwdu + +warnings.simplefilter('always', DeprecationWarning) + + +class cached_property(property): + """A property caching its return value + + This is pretty much the same as a normal Python property, except that the + decorated function is called only once. Its return value is then saved, + subsequent calls will return it without executing the function any more. + + Example: + >>> class Foo(object): + ... @cached_property + ... def bar(self): + ... print("Executing Foo.bar...") + ... return 42 + ... + >>> f = Foo() + >>> f.bar + Executing Foo.bar... + 42 + >>> f.bar + 42 + """ + def __get__(self, inst, type=None): + try: + return getattr(inst, '_%s' % self.fget.__name__) + except AttributeError: + v = super(cached_property, self).__get__(inst, type) + setattr(inst, '_%s' % self.fget.__name__, v) + return v + + +def warn_deprecated(clsname, oldname, newname): + """Emit a deprecation warning + + Args: + clsname (str): The name of the class which has its attribute + deprecated. + oldname (str): The name of the deprecated attribute. + newname (str): The name of the new attribute, which should be used + instead. + """ + warnings.warn( + "%s.%s is deprecated and will be removed eventually.\n Please " + "use %s.%s instead." % (clsname, oldname, clsname, newname), + DeprecationWarning, stacklevel=3) + + +def _log_value(log_func, value, level, indent, suffix=''): + offset = ' ' * level * indent + log_func(''.join([offset, str(value), suffix])) + + +def log_result(log_func, result, level=0, indent=2): + if isinstance(result, list): + for item in result: + log_result(log_func, item, level) + elif isinstance(result, dict): + for key, value in result.items(): + _log_value(log_func, key, level, indent, ':') + log_result(log_func, value, level+1) + else: + _log_value(log_func, result, level, indent) diff --git a/setup.cfg b/setup.cfg index f1d21b6..02757bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,4 +10,3 @@ detailed-errors = 1 with-coverage = 1 cover-package = pyrpkg cover-erase = 1 -tests=../test/ \ No newline at end of file diff --git a/setup.py b/setup.py index 8b410df..15d0ba7 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/python -from setuptools import setup +from setuptools import setup, find_packages setup( @@ -12,11 +12,10 @@ setup( "package sources in a git repository"), license="GPLv2+", url="https://fedorahosted.org/rpkg", - package_dir={'': 'src'}, - packages=['pyrpkg'], - scripts=['src/rpkg'], - data_files=[('/etc/bash_completion.d', ['src/rpkg.bash']), - ('/etc/rpkg', ['src/rpkg.conf'])], + packages=find_packages(), + scripts=['bin/rpkg'], + data_files=[('/etc/bash_completion.d', ['etc/bash_completion.d/rpkg.bash']), + ('/etc/rpkg', ['etc/rpkg/rpkg.conf'])], install_requires=['six', 'pycurl'], # + koji, but it's not in PyPI tests_require=['nose', 'mock'], test_suite='nose.collector', diff --git a/src/pyrpkg/__init__.py b/src/pyrpkg/__init__.py deleted file mode 100644 index 17abde0..0000000 --- a/src/pyrpkg/__init__.py +++ /dev/null @@ -1,2590 +0,0 @@ -# pyrpkg - a Python library for RPM Packagers -# -# Copyright (C) 2011 Red Hat Inc. -# Author(s): Jesse Keating -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - -import errno -import fnmatch -import git -import glob -import koji -import logging -import os -import posixpath -import pwd -import re -import rpm -import shutil -import six -import sys -import tempfile - -from ConfigParser import ConfigParser - -from osbs.api import OSBS -from osbs.conf import Configuration -from six.moves import configparser -from six.moves import urllib - -from pyrpkg.errors import HashtypeMixingError, rpkgError, rpkgAuthError, \ - UnknownTargetError -from .gitignore import GitIgnore -from pyrpkg.lookaside import CGILookasideCache -from pyrpkg.sources import SourcesFile -from pyrpkg.utils import cached_property, log_result - -if sys.version_info[0:2] >= (2, 5): - import subprocess -else: - # We need a subprocess that has check_call - from kitchen.pycompat27 import subprocess - -# Try to import krb, it's OK if it fails -try: - import krbV -except ImportError: - pass - - -class NullHandler(logging.Handler): - """Null logger to avoid spurious messages, add a handler in app code""" - def emit(self, record): - pass - - -h = NullHandler() -# This is our log object, clients of this library can use this object to -# define their own logging needs -log = logging.getLogger(__name__) -# Add the null handler -log.addHandler(h) - - -class Commands(object): - """This is a class to hold all the commands that will be called - by clients - """ - - # This shouldn't change... often - UPLOADEXTS = ['tar', 'gz', 'bz2', 'lzma', 'xz', 'Z', 'zip', 'tff', - 'bin', 'tbz', 'tbz2', 'tgz', 'tlz', 'txz', 'pdf', 'rpm', - 'jar', 'war', 'db', 'cpio', 'jisp', 'egg', 'gem', 'spkg', - 'oxt', 'xpi'] - - def __init__(self, path, lookaside, lookasidehash, lookaside_cgi, - gitbaseurl, anongiturl, branchre, kojiconfig, - build_client, user=None, - dist=None, target=None, quiet=False, - distgit_namespaced=False): - """Init the object and some configuration details.""" - - # Path to operate on, most often pwd - self._path = None - self.path = os.path.abspath(path) - # The url of the lookaside for source archives - self.lookaside = lookaside - # The type of hash to use with the lookaside - self.lookasidehash = lookasidehash - # The CGI server for the lookaside - self.lookaside_cgi = lookaside_cgi - # The base URL of the git server - self.gitbaseurl = gitbaseurl - # The anonymous version of the git url - self.anongiturl = anongiturl - # The regex of branches we care about - self.branchre = branchre - # The location of the buildsys config file - self.kojiconfig = os.path.expanduser(kojiconfig) - # The buildsys client to use - self.build_client = build_client - # A way to override the discovered "distribution" - self.dist = dist - # Set the default hashtype - self.hashtype = 'sha256' - # Set an attribute for quiet or not - self.quiet = quiet - # Set place holders for properties - # Anonymous buildsys session - self._anon_kojisession = None - # The upstream branch a downstream branch is tracking - self._branch_merge = None - # The latest commit - self._commit = None - # The disttag rpm value - self._disttag = None - # The distval rpm value - self._distval = None - # The distvar rpm value - self._distvar = None - # The rpm epoch of the cloned module - self._epoch = None - # An authenticated buildsys session - self._kojisession = None - # A web url of the buildsys server - self._kojiweburl = None - # The local arch to use in rpm building - self._localarch = None - # A property to load the mock config - self._mockconfig = None - # The name of the cloned module - self._module_name = None - # The distgit namespaced name of the cloned module - self._ns_module_name = None - # The name of the module from spec file - self._module_name_spec = None - # The rpm name-version-release of the cloned module - self._nvr = None - # The rpm release of the cloned module - self._rel = None - # The cloned repo object - self._repo = None - # The rpm defines used when calling rpm - self._rpmdefines = None - # The specfile in the cloned module - self._spec = None - # The build target within the buildsystem - self._target = target - # The top url to our build server - self._topurl = None - # The user to use or discover - self._user = user - # The password to use - self._password = None - # The alternate Koji user to run commands as - self._runas = None - # The rpm version of the cloned module - self._ver = None - self.log = log - # Pushurl or url of remote of branch - self._push_url = None - # Name of remote determined from current clone - self._branch_remote = None - # Name of default remote to be used for new clone - self.default_branch_remote = 'origin' - # Default sources file output format type - self.source_entry_type = 'old' - # Set an attribute debug - self.debug = False - # Set an attribute verbose - self.verbose = False - # Config to set after cloning - self.clone_config = None - # Git namespacing for more than just rpm build artifacts - self.distgit_namespaced = distgit_namespaced - - # Define properties here - # Properties allow us to "lazy load" various attributes, which also means - # that we can do clone actions without knowing things like the spec - # file or rpm data. - - @cached_property - def lookasidecache(self): - """A helper to interact with the lookaside cache - - This is a pyrpkg.lookaside.CGILookasideCache instance, providing all - the needed stuff to communicate with a Fedora-style lookaside cache. - - Downstream users of the pyrpkg API may override this property with - their own, returning their own implementation of a lookaside cache - helper object. - """ - return CGILookasideCache( - self.lookasidehash, self.lookaside, self.lookaside_cgi, - client_cert=self.cert_file, ca_cert=self.ca_cert) - - @property - def path(self): - return self._path - - @path.setter - def path(self, value): - if self._path != value: - # Ensure all properties which depend on self.path will be - # freshly loaded next time - self._push_url = None - self._branch_remote = None - self._repo = None - self._ns_module_name = None - self._path = value - - @property - def anon_kojisession(self): - """This property ensures the anon kojisession attribute""" - - if not self._anon_kojisession: - self.load_kojisession(anon=True) - return self._anon_kojisession - - def load_kojisession(self, anon=False): - """Initiate a koji session. - - The koji session can be logged in or anonymous - """ - - # Stealing a bunch of code from /usr/bin/koji here, too bad it isn't - # in a more usable library form - defaults = { - 'server': None, - 'topurl': 'http://localhost/kojiroot', - 'weburl': 'http://localhost/koji', - 'cert': '~/.koji/client.crt', - 'ca': '~/.koji/clientca.crt', - 'serverca': '~/.koji/serverca.crt', - 'authtype': None, - 'krbservice': None, - 'timeout': None, - 'keepalive': True, - 'max_retries': None, - 'retry_interval': None, - 'anon_retry': True, - 'offline_retry': None, - 'offline_retry_interval': None, - 'use_fast_upload': None, - 'debug': None, - 'debug_xmlrpc': None - } - - # Process the configs in order, global, user, then any option passed - config = configparser.ConfigParser() - confs = [self.kojiconfig, - os.path.expanduser('~/.koji/config')] - config.read(confs) - - if config.has_section(os.path.basename(self.build_client)): - for name, value in config.items(os.path.basename( - self.build_client)): - if name in defaults: - if name in ('keepalive', 'anon_retry', 'offline_retry', - 'use_fast_upload', - 'debug', 'debug_xmlrpc'): - defaults[name] = config.getboolean( - os.path.basename(self.build_client), name) - elif name in ('timeout', 'max_retries', 'retry_interval', - 'offline_retry_interval'): - defaults[name] = config.getint( - os.path.basename(self.build_client), name) - else: - defaults[name] = value - if not defaults['server']: - raise rpkgError('No server defined in: %s' % ', '.join(confs)) - # Expand out the directory options - for name in ('cert', 'ca', 'serverca'): - if defaults[name]: - defaults[name] = os.path.expanduser(defaults[name]) - self.log.debug('Initiating a %s session to %s', - os.path.basename(self.build_client), defaults['server']) - session_opts = {} - for name in ('krbservice', 'timeout', 'keepalive', - 'max_retries', 'retry_interval', 'anon_retry', - 'offline_retry', 'offline_retry_interval', - 'debug', 'debug_xmlrpc', - 'use_fast_upload'): - if defaults[name] is not None: - session_opts[name] = defaults[name] - try: - if anon: - self._anon_kojisession = koji.ClientSession(defaults['server'], - session_opts) - else: - self._kojisession = koji.ClientSession(defaults['server'], - session_opts) - except: - raise rpkgError('Could not initiate %s session' % - os.path.basename(self.build_client)) - # save the weburl and topurl for later use as well - self._kojiweburl = defaults['weburl'] - self._topurl = defaults['topurl'] - if not anon: - # Default to ssl if not otherwise specified and we have the cert - if defaults['authtype'] == 'ssl' or \ - os.path.isfile(defaults['cert']) and \ - defaults['authtype'] is None: - try: - self._kojisession.ssl_login(defaults['cert'], - defaults['ca'], - defaults['serverca'], - proxyuser=self.runas) - except koji.ssl.SSLCommon.SSL.Error as error: - for (_, _, ssl_reason) in error.message: - # Use heuristic. Some OpenSSL libs doesn't store error - # codes - if 'certificate revoked' in ssl_reason or \ - 'certificate expired' in ssl_reason: - self.log.info("Certificate is revoked or expired.") - raise rpkgAuthError('Could not auth with koji. Login ' - 'failed: %s' % error) - # Or try password auth - elif defaults['authtype'] == 'password' or self.password \ - and defaults['authtype'] is None: - if self.runas: - raise rpkgError('--runas cannot be used with password auth') - self._kojisession.opts['user'] = self.user - self._kojisession.opts['password'] = self.password - self._kojisession.login() - # Or try kerberos - elif defaults['authtype'] == 'kerberos' or self._has_krb_creds() \ - and defaults['authtype'] is None: - self._kojisession.krb_login(proxyuser=self.runas) - if not self._kojisession.logged_in: - raise rpkgError('Could not login to %s' % defaults['server']) - - @property - def branch_merge(self): - """This property ensures the branch attribute""" - - if not self._branch_merge: - self.load_branch_merge() - return(self._branch_merge) - - def load_branch_merge(self): - """Find the remote tracking branch from the branch we're on. - - The goal of this function is to catch if we are on a branch we - - can make some assumptions about. If there is no merge point - - then we raise and ask the user to specify. - """ - - if self.dist: - self._branch_merge = self.dist - else: - try: - localbranch = self.repo.active_branch.name - except TypeError as e: - raise rpkgError('Repo in inconsistent state: %s' % e) - try: - merge = self.repo.git.config('--get', - 'branch.%s.merge' % localbranch) - except git.GitCommandError as e: - raise rpkgError('Unable to find remote branch. Use --dist') - # Trim off the refs/heads so that we're just working with - # the branch name - merge = merge.replace('refs/heads/', '') - self._branch_merge = merge - - @property - def branch_remote(self): - """This property ensures the branch_remote attribute""" - - if not self._branch_remote: - self.load_branch_remote() - return self._branch_remote - - def load_branch_remote(self): - """Find the name of remote from branch we're on.""" - - try: - remote = self.repo.git.config('--get', 'branch.%s.remote' - % self.branch_merge) - except (git.GitCommandError, rpkgError) as e: - remote = self.default_branch_remote - self.log.debug("Could not determine the remote name: %s", str(e)) - self.log.debug("Falling back to default remote name '%s'", remote) - - self._branch_remote = remote - - @property - def push_url(self): - """This property ensures the push_url attribute""" - - if not self._push_url: - self.load_push_url() - return self._push_url - - def load_push_url(self): - """Find the pushurl or url of remote of branch we're on.""" - try: - url = self.repo.git.remote('get-url', '--push', self.branch_remote) - except git.GitCommandError as e: - try: - url = self.repo.git.config( - '--get', 'remote.%s.pushurl' % self.branch_remote) - except git.GitCommandError as e: - try: - url = self.repo.git.config( - '--get', 'remote.%s.url' % self.branch_remote) - except git.GitCommandError as e: - raise rpkgError('Unable to find remote push url: %s' % e) - if isinstance(url, six.text_type): - # GitPython >= 1.0 return unicode. It must be encoded to string. - self._push_url = url.encode('utf-8') - else: - self._push_url = url - - @property - def commithash(self): - """This property ensures the commit attribute""" - - if not self._commit: - self.load_commit() - return self._commit - - def load_commit(self): - """Discover the latest commit to the package""" - - # Get the commit hash - comobj = six.next(self.repo.iter_commits()) - # Work around different versions of GitPython - if hasattr(comobj, 'sha'): - self._commit = comobj.sha - else: - self._commit = comobj.hexsha - - @property - def disttag(self): - """This property ensures the disttag attribute""" - - if not self._disttag: - self.load_rpmdefines() - return self._disttag - - @property - def distval(self): - """This property ensures the distval attribute""" - - if not self._distval: - self.load_rpmdefines() - return self._distval - - @property - def distvar(self): - """This property ensures the distvar attribute""" - - if not self._distvar: - self.load_rpmdefines() - return self._distvar - - @property - def epoch(self): - """This property ensures the epoch attribute""" - - if not self._epoch: - self.load_nameverrel() - return self._epoch - - @property - def kojisession(self): - """This property ensures the kojisession attribute""" - - if not self._kojisession: - self.load_kojisession() - return self._kojisession - - @property - def kojiweburl(self): - """This property ensures the kojiweburl attribute""" - - if not self._kojiweburl: - self.load_kojisession() - return self._kojiweburl - - @property - def localarch(self): - """This property ensures the module attribute""" - - if not self._localarch: - self.load_localarch() - return(self._localarch) - - def load_localarch(self): - """Get the local arch as defined by rpm""" - - proc = subprocess.Popen(['rpm --eval %{_arch}'], shell=True, - stdout=subprocess.PIPE) - self._localarch = proc.communicate()[0].strip('\n') - - @property - def mockconfig(self): - """This property ensures the mockconfig attribute""" - - if not self._mockconfig: - self.load_mockconfig() - return self._mockconfig - - @mockconfig.setter - def mockconfig(self, config): - self._mockconfig = config - - def load_mockconfig(self): - """This sets the mockconfig attribute""" - - self._mockconfig = '%s-%s' % (self.target, self.localarch) - - @property - def module_name(self): - """This property ensures the module attribute""" - - if not self._module_name: - self.load_module_name() - return self._module_name - - @module_name.setter - def module_name(self, module_name): - self._module_name = module_name - - def load_module_name(self): - """Loads a package module.""" - - try: - if self.push_url: - parts = urllib.parse.urlparse(self.push_url) - - # FIXME - # if self.distgit_namespaced: - # self._module_name = "/".join(parts.path.split("/")[-2:]) - module_name = posixpath.basename(parts.path) - - if module_name.endswith(b'.git'): - module_name = module_name[:-len(b'.git')] - self._module_name = module_name - return - except rpkgError: - self.log.warning('Failed to get module name from Git url or pushurl') - - self.load_nameverrel() - if self._module_name_spec: - self._module_name = self._module_name_spec - return - - raise rpkgError('Could not find current module name.' - ' Use --module-name.') - - @property - def ns_module_name(self): - """This property ensures the module attribute""" - - if not self._ns_module_name: - self.load_ns_module_name() - return self._ns_module_name - - @ns_module_name.setter - def ns_module_name(self, ns_module_name): - self._ns_module_name = ns_module_name - - def _print_old_checkout_warning(self, module): - self.log.warning('Your git configuration does not use a namespace.') - self.log.warning('Consider updating your git configuration by running:') - self.log.warning(' git remote set-url %s %s', - self.branch_remote, self._get_namespace_giturl(module)) - - def load_ns_module_name(self): - """Loads a package module.""" - - try: - if self.push_url: - parts = urllib.parse.urlparse(self.push_url) - - if self.distgit_namespaced: - path_parts = [p for p in parts.path.split("/") if p] - if len(path_parts) == 1: - self._print_old_checkout_warning(path_parts[0]) - path_parts.insert(0, "rpms") - ns_module_name = "/".join(path_parts[-2:]) - else: - ns_module_name = posixpath.basename(parts.path) - - if ns_module_name.endswith('.git'): - ns_module_name = ns_module_name[:-len('.git')] - self._ns_module_name = ns_module_name - return - except rpkgError: - self.log.warning('Failed to get ns_module_name from Git url or pushurl') - - @property - def nvr(self): - """This property ensures the nvr attribute""" - - if not self._nvr: - self.load_nvr() - return self._nvr - - def load_nvr(self): - """This sets the nvr attribute""" - - self._nvr = '%s-%s-%s' % (self.module_name, self.ver, self.rel) - - @property - def rel(self): - """This property ensures the rel attribute""" - if not self._rel: - self.load_nameverrel() - return(self._rel) - - def load_nameverrel(self): - """Set the release of a package module.""" - - cmd = ['rpm'] - cmd.extend(self.rpmdefines) - # We make sure there is a space at the end of our query so that - # we can split it later. When there are subpackages, we get a - # listing for each subpackage. We only care about the first. - cmd.extend(['-q', '--qf', '"%{NAME} %{EPOCH} %{VERSION} %{RELEASE}??"', - '--specfile', '"%s"' % os.path.join(self.path, self.spec)]) - joined_cmd = ' '.join(cmd) - try: - proc = subprocess.Popen(joined_cmd, shell=True, - universal_newlines=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, err = proc.communicate() - except Exception as e: - if err: - self.log.debug('Errors occoured while running following command to get N-V-R-E:') - self.log.debug(joined_cmd) - self.log.error(err) - raise rpkgError('Could not query n-v-r of %s: %s' - % (self.module_name, e)) - if err: - self.log.debug('Errors occoured while running following command to get N-V-R-E:') - self.log.debug(joined_cmd) - self.log.error(err) - # Get just the output, then split it by ??, grab the first and split - # again to get ver and rel - first_line_output = output.split('??')[0] - parts = first_line_output.split() - if len(parts) != 4: - raise rpkgError('Could not get n-v-r-e from %r' - % first_line_output) - (self._module_name_spec, - self._epoch, - self._ver, - self._rel) = parts - - # Most packages don't include a "Epoch: 0" line, in which case RPM - # returns '(none)' - if self._epoch == "(none)": - self._epoch = "0" - - @property - def repo(self): - """This property ensures the repo attribute""" - - if not self._repo: - self.load_repo() - return(self._repo) - - def load_repo(self): - """Create a repo object from our path""" - - self.log.debug('Creating repo object from %s', self.path) - try: - self._repo = git.Repo(self.path) - except git.InvalidGitRepositoryError: - raise rpkgError('%s is not a valid repo' % self.path) - - @property - def rpmdefines(self): - """This property ensures the rpm defines""" - - if not self._rpmdefines: - self.load_rpmdefines() - return(self._rpmdefines) - - def load_rpmdefines(self): - """Populate rpmdefines based on current active branch""" - - # This is another function ripe for subclassing - - try: - # This regex should find the 'rhel-5' or 'rhel-6.2' parts of the - # branch name. There should only be one of those, and all branches - # should end in one. - osver = re.search(r'rhel-\d.*$', self.branch_merge).group() - except AttributeError: - raise rpkgError('Could not find the base OS ver from branch name' - ' %s. Consider using --dist option' % - self.branch_merge) - self._distvar, self._distval = osver.split('-') - self._distval = self._distval.replace('.', '_') - self._disttag = 'el%s' % self._distval - self._rpmdefines = ["--define '_sourcedir %s'" % self.path, - "--define '_specdir %s'" % self.path, - "--define '_builddir %s'" % self.path, - "--define '_srcrpmdir %s'" % self.path, - "--define '_rpmdir %s'" % self.path, - "--define 'dist .%s'" % self._disttag, - "--define '%s %s'" % (self._distvar, - self._distval.split('_')[0]), - # int and float this to remove the decimal - "--define '%s 1'" % self._disttag] - - @property - def spec(self): - """This property ensures the module attribute""" - - if not self._spec: - self.load_spec() - return self._spec - - def load_spec(self): - """This sets the spec attribute""" - - deadpackage = False - - # Get a list of files in the path we're looking at - files = os.listdir(self.path) - # Search the files for the first one that ends with ".spec" - for f in files: - if f.endswith('.spec') and not f.startswith('.'): - self._spec = f - return - if f == 'dead.package': - deadpackage = True - if deadpackage: - raise rpkgError('No spec file found. This package is retired') - else: - raise rpkgError('No spec file found.') - - @property - def target(self): - """This property ensures the target attribute""" - - if not self._target: - self.load_target() - return self._target - - def load_target(self): - """This creates the target attribute based on branch merge""" - - # If a site has a different naming scheme, this would be where - # a site would override - self._target = '%s-candidate' % self.branch_merge - - @property - def topurl(self): - """This property ensures the topurl attribute""" - - if not self._topurl: - # Assume anon here, whatever. - self.load_kojisession(anon=True) - return self._topurl - - @property - def user(self): - """This property ensures the user attribute""" - - if not self._user: - self.load_user() - return self._user - - def load_user(self): - """This sets the user attribute""" - - # If a site figures out the user differently (like from ssl cert) - # this is where you'd override and make that happen - self._user = pwd.getpwuid(os.getuid())[0] - - @property - def password(self): - """This property ensures the password attribute""" - - return self._password - - @password.setter - def password(self, password): - self._password = password - - @property - def runas(self): - """This property ensures the runas attribute""" - - return self._runas - - @runas.setter - def runas(self, runas): - self._runas = runas - - @property - def ver(self): - """This property ensures the ver attribute""" - if not self._ver: - self.load_nameverrel() - return(self._ver) - - @property - def mock_results_dir(self): - return os.path.join(self.path, "results_%s" % self.module_name, - self.ver, self.rel) - - @property - def sources_filename(self): - return os.path.join(self.path, 'sources') - - @property - def osbs_config_filename(self): - return os.path.join(self.path, '.osbs-repo-config') - - @property - def cert_file(self): - """A client-side certificate for SSL authentication - - Downstream users of the pyrpkg API should override this property if - they actually need to use a client-side certificate. - - This defaults to None, which means no client-side certificate is used. - """ - return None - - @property - def ca_cert(self): - """A CA certificate to authenticate the server in SSL connections - - Downstream users of the pyrpkg API should override this property if - they actually need to use a CA certificate, usually because their - lookaside cache is using HTTPS with a self-signed certificate. - - This defaults to None, which means the system CA bundle is used. - """ - return None - - # Define some helper functions, they start with _ - def _has_krb_creds(self): - # This function is lifted from /usr/bin/koji - if 'krbV' not in sys.modules: - return False - try: - ctx = krbV.default_context() - ccache = ctx.default_ccache() - princ = ccache.principal() # noqa - return True - except krbV.Krb5Error: - return False - - def _run_command(self, cmd, shell=False, env=None, pipe=[], cwd=None): - """Run the given command. - - _run_command is able to run single command or two commands via pipe. - Whatever the way to run the command, output to both stdout and stderr - will not be captured and output to terminal directly, that is useful - for caller to redirect. - - cmd is a list of the command and arguments - - shell is whether to run in a shell or not, defaults to False - - env is a dict of environment variables to use (if any) - - pipe is a command to pipe the output of cmd into - - cwd is the optional directory to run the command from - - Raises on error, or returns nothing. - """ - - # Process any environment variables. - environ = os.environ - if env: - for item in env.keys(): - self.log.debug('Adding %s:%s to the environment', item, env[item]) - environ[item] = env[item] - # Check if we're supposed to be on a shell. If so, the command must - # be a string, and not a list. - command = cmd - pipecmd = pipe - if shell: - command = ' '.join(cmd) - pipecmd = ' '.join(pipe) - - if pipe: - self.log.debug('Running: %s | %s', ' '.join(cmd), ' '.join(pipe)) - else: - self.log.debug('Running: %s', ' '.join(cmd)) - - try: - if pipe: - # We're piping the stderr over as well, which is probably a - # bad thing, but rpmbuild likes to put useful data on - # stderr, so.... - proc = subprocess.Popen(command, env=environ, shell=shell, cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - subprocess.check_call(pipecmd, env=environ, shell=shell, cwd=cwd, stdin=proc.stdout) - else: - subprocess.check_call(command, env=environ, shell=shell, cwd=cwd) - except (subprocess.CalledProcessError, OSError) as e: - raise rpkgError(e) - except KeyboardInterrupt: - raise rpkgError('Command is terminated by user.') - except Exception as e: - raise rpkgError(e) - - def _newer(self, file1, file2): - """Compare the last modification time of the given files - - Returns True is file1 is newer than file2 - - """ - - return os.path.getmtime(file1) > os.path.getmtime(file2) - - def _get_build_arches_from_spec(self): - """Given the path to an spec, retrieve the build arches - - """ - - spec = os.path.join(self.path, self.spec) - try: - hdr = rpm.spec(spec) - except Exception: - raise rpkgError('%s is not a spec file' % spec) - archlist = [pkg.header['arch'] for pkg in hdr.packages] - if not archlist: - raise rpkgError('No compatible build arches found in %s' % spec) - return archlist - - def _get_build_arches_from_srpm(self, srpm, arches): - """Given the path to an srpm, determine the possible build arches - - Use supplied arches as a filter, only return compatible arches - - """ - - archlist = arches - hdr = koji.get_rpm_header(srpm) - if hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1: - raise rpkgError('%s is not a source package.' % srpm) - buildarchs = hdr[rpm.RPMTAG_BUILDARCHS] - exclusivearch = hdr[rpm.RPMTAG_EXCLUSIVEARCH] - excludearch = hdr[rpm.RPMTAG_EXCLUDEARCH] - # Reduce by buildarchs - if buildarchs: - archlist = [a for a in archlist if a in buildarchs] - # Reduce by exclusive arches - if exclusivearch: - archlist = [a for a in archlist if a in exclusivearch] - # Reduce by exclude arch - if excludearch: - archlist = [a for a in archlist if a not in excludearch] - # do the noarch thing - if 'noarch' not in excludearch and ('noarch' in buildarchs or - 'noarch' in exclusivearch): - archlist.append('noarch') - # See if we have anything compatible. Should we raise here? - if not archlist: - raise rpkgError('No compatible build arches found in %s' % srpm) - return archlist - - def _guess_hashtype(self): - """Attempt to figure out the hash type based on branch data""" - - # We may not be able to determine the rpmdefine, if so, fall back. - try: - # This works, except for the small range of Fedoras - # between FC5 and FC12 or so. Nobody builds for that old - # anyway. - if int(re.search(r'\d+', self.distval).group()) < 6: - return('md5') - except: - # An error here is OK, don't bother the user. - pass - - # Fall back to the default hash type - return(self.hashtype) - - def _fetch_remotes(self): - self.log.debug('Fetching remotes') - for remote in self.repo.remotes: - self.repo.git.fetch(remote) - - def _list_branches(self, fetch=True): - """Returns a tuple of local and remote branch names""" - - if fetch: - self._fetch_remotes() - self.log.debug('Listing refs') - refs = self.repo.refs - # Sort into local and remote branches - remotes = [] - locals = [] - for ref in refs: - if type(ref) == git.Head: - self.log.debug('Found local branch %s', ref.name) - locals.append(ref.name) - elif type(ref) == git.RemoteReference: - if ref.remote_head == 'HEAD': - self.log.debug('Skipping remote branch alias HEAD') - continue # Not useful in this context - self.log.debug('Found remote branch %s', ref.name) - remotes.append(ref.name) - return (locals, remotes) - - def _srpmdetails(self, srpm): - """Return a tuple of package name, package files, and upload files.""" - - # get the name - cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME}', srpm] - # Run the command - self.log.debug('Running: %s', ' '.join(cmd)) - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = proc.communicate() - except OSError as e: - raise rpkgError(e) - name = output - if error: - raise rpkgError('Error querying srpm: %s' % error) - - # now get the files and upload files - files = [] - uploadfiles = [] - cmd = ['rpm', '-qpl', srpm] - self.log.debug('Running: %s', ' '.join(cmd)) - env = dict(os.environ) - env["LANG"] = "C" - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env) - output, error = proc.communicate() - except OSError as e: - raise rpkgError(e) - # work around signed SRPMs, for these rpm -qpl might print a warning - # like: - # warning: foo-0.0.src.rpm Header V3 RSA/SHA256 Signature, key ID - # fd431d51: NOKEY - if error and not error.startswith("warning:") and "NOKEY" not in error: - raise rpkgError('Error querying srpm: %s' % error) - contents = output.strip().split('\n') - # Cycle through the stuff and sort correctly by its extension - for file in contents: - if file.rsplit('.')[-1] in self.UPLOADEXTS: - uploadfiles.append(file) - else: - files.append(file) - - return((name, files, uploadfiles)) - - def _get_namespace_giturl(self, module): - """Get the namespaced git url, if DistGit namespaces enabled - - Takes a module name - - Returns a string of giturl - - """ - - if self.distgit_namespaced: - if '/' in module: - giturl = self.gitbaseurl % \ - {'user': self.user, 'module': module} - else: - # Default to rpms namespace for backwards compat - giturl = self.gitbaseurl % \ - {'user': self.user, 'module': "rpms/%s" % module} - else: - giturl = self.gitbaseurl % \ - {'user': self.user, 'module': module} - - return giturl - - def _get_namespace_anongiturl(self, module): - """Get the namespaced git url, if DistGit namespaces enabled - - Takes a module name - - Returns a string of giturl - - """ - - if self.distgit_namespaced: - if '/' in module: - giturl = self.anongiturl % {'module': module} - else: - # Default to rpms namespace for backwards compat - giturl = self.anongiturl % {'module': "rpms/%s" % module} - else: - giturl = self.anongiturl % {'module': module} - - return giturl - - def add_tag(self, tagname, force=False, message=None, file=None): - """Add a git tag to the repository - - Takes a tagname - - Optionally can force the tag, include a message, - or reference a message file. - - Runs the tag command and returns nothing - - """ - - cmd = ['git', 'tag'] - cmd.extend(['-a']) - # force tag creation, if tag already exists - if force: - cmd.extend(['-f']) - # Description for the tag - if message: - cmd.extend(['-m', message]) - elif file: - cmd.extend(['-F', os.path.abspath(file)]) - cmd.append(tagname) - # make it so - self._run_command(cmd, cwd=self.path) - self.log.info('Tag \'%s\' was created', tagname) - - def clean(self, dry=False, useignore=True): - """Clean a module checkout of untracked files. - - Can optionally perform a dry-run - - Can optionally not use the ignore rules - - Logs output and returns nothing - - """ - - # setup the command, this could probably be done with some python api... - cmd = ['git', 'clean', '-f', '-d'] - if dry: - cmd.append('--dry-run') - if not useignore: - cmd.append('-x') - if self.quiet: - cmd.append('-q') - # Run it! - self._run_command(cmd, cwd=self.path) - return - - def clone(self, module, path=None, branch=None, bare_dir=None, - anon=False, target=None): - """Clone a repo, optionally check out a specific branch. - - module is the name of the module to clone - - path is the basedir to perform the clone in - - branch is the name of a branch to checkout instead of /master - - bare_dir is the name of a directory to make a bare clone to, if this - is a bare clone. None otherwise. - - anon is whether or not to clone anonymously - - target is the name of the folder in which to clone the repo - - Logs the output and returns nothing. - - """ - - if not path: - path = self.path - self._push_url = None - self._branch_remote = None - # construct the git url - if anon: - giturl = self._get_namespace_anongiturl(module) - else: - giturl = self._get_namespace_giturl(module) - - # Create the command - cmd = ['git', 'clone'] - if self.quiet: - cmd.append('-q') - # do the clone - if branch and bare_dir: - raise rpkgError('Cannot combine bare cloning with a branch') - elif branch: - # For now we have to use switch branch - self.log.debug('Checking out a specific branch %s', giturl) - cmd.extend(['-b', branch, giturl]) - elif bare_dir: - self.log.debug('Cloning %s bare', giturl) - cmd.extend(['--bare', giturl]) - if not target: - cmd.append(bare_dir) - else: - self.log.debug('Cloning %s', giturl) - cmd.extend([giturl]) - - if not bare_dir: - # --bare and --origin are incompatible - cmd.extend(['--origin', self.default_branch_remote]) - - if target: - self.log.debug('Cloning into: %s', target) - cmd.append(target) - - self._run_command(cmd, cwd=path) - - if self.clone_config: - base_module = self.get_base_module(module) - git_dir = target if target else bare_dir if bare_dir else base_module - conf_git = git.Git(os.path.join(path, git_dir)) - self._clone_config(conf_git, module) - - return - - def get_base_module(self, module): - # Handle namespaced modules - # Example: - # module: docker/cockpit - # The path will just be os.path.join(path, "cockpit") - if "/" in module: - return module.split("/")[-1] - return module - - def clone_with_dirs(self, module, anon=False, target=None): - """Clone a repo old style with subdirs for each branch. - - module is the name of the module to clone - - gitargs is an option list of arguments to git clone - - """ - - self._push_url = None - self._branch_remote = None - # Get the full path of, and git object for, our directory of branches - top_path = os.path.join(self.path, - target or self.get_base_module(module)) - top_git = git.Git(top_path) - repo_path = os.path.join(top_path, 'rpkg.git') - - # construct the git url - if anon: - giturl = self._get_namespace_anongiturl(module) - else: - giturl = self._get_namespace_giturl(module) - - # Create our new top directory - try: - os.mkdir(top_path) - except OSError as e: - raise rpkgError('Could not create directory for module %s: %s' - % (module, e)) - - # Create a bare clone first. This gives us a good list of branches - try: - self.clone(module, top_path, bare_dir=repo_path, anon=anon) - except Exception as e: - # Clean out our directory - shutil.rmtree(top_path) - raise - # Get the full path to, and a git object for, our new bare repo - repo_git = git.Git(repo_path) - - # Get a branch listing - branches = [x for x in repo_git.branch().split() - if x != "*" and re.search(self.branchre, x)] - - for branch in branches: - try: - # Make a local clone for our branch - top_git.clone("--branch", branch, - "--origin", self.default_branch_remote, - repo_path, branch) - - # Set the origin correctly - branch_path = os.path.join(top_path, branch) - branch_git = git.Git(branch_path) - branch_git.config("--replace-all", - "remote.%s.url" % self.default_branch_remote, - giturl) - except (git.GitCommandError, OSError) as e: - raise rpkgError('Could not locally clone %s from %s: %s' - % (branch, repo_path, e)) - - # We don't need this now. Ignore errors since keeping it does no harm - shutil.rmtree(repo_path, ignore_errors=True) - - def _clone_config(self, conf_git, module): - clone_config = self.clone_config.strip() % {'module': module} - for confline in clone_config.splitlines(): - if confline: - conf_git.config(*confline.split()) - - def commit(self, message=None, file=None, files=[], signoff=False): - """Commit changes to a module (optionally found at path) - - Can take a message to use as the commit message - - a file to find the commit message within - - and a list of files to commit. - - Requires the caller be a real tty or a message passed. - - Logs the output and returns nothing. - - """ - - # First lets see if we got a message or we're on a real tty: - if not sys.stdin.isatty(): - if not message and not file: - raise rpkgError('Must have a commit message or be on a real ' - 'tty.') - - # construct the git command - # We do this via subprocess because the git module is terrible. - cmd = ['git', 'commit'] - if signoff: - cmd.append('-s') - if self.quiet: - cmd.append('-q') - if message: - cmd.extend(['-m', message]) - elif file: - # If we get a relative file name, prepend our path to it. - if self.path and not file.startswith('/'): - cmd.extend(['-F', os.path.abspath(os.path.join(self.path, - file))]) - else: - cmd.extend(['-F', os.path.abspath(file)]) - if not files: - cmd.append('-a') - else: - cmd.extend(files) - # make it so - self._run_command(cmd, cwd=self.path) - return - - def delete_tag(self, tagname): - """Delete a git tag from the repository found at optional path""" - - try: - self.repo.delete_tag(tagname) - - except git.GitCommandError as e: - raise rpkgError(e) - - self.log.info('Tag %s was deleted', tagname) - - def diff(self, cached=False, files=[]): - """Execute a git diff - - optionally diff the cached or staged changes - - Takes an optional list of files to diff relative to the module base - directory - - Logs the output and returns nothing - - """ - - # Things work better if we're in our module directory - oldpath = os.getcwd() - os.chdir(self.path) - # build up the command - cmd = ['git', 'diff'] - if cached: - cmd.append('--cached') - if files: - cmd.extend(files) - - # Run it! - self._run_command(cmd) - # popd - os.chdir(oldpath) - return - - def get_latest_commit(self, module, branch): - """Discover the latest commit has for a given module and return it""" - - # This is stupid that I have to use subprocess :/ - url = self._get_namespace_anongiturl(module) - # This cmd below only works to scratch build rawhide - # We need something better for epel - cmd = ['git', 'ls-remote', url, 'refs/heads/%s' % branch] - try: - proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, - stdout=subprocess.PIPE) - output, error = proc.communicate() - except OSError as e: - raise rpkgError(e) - if error: - raise rpkgError('Got an error finding %s head for %s: %s' - % (branch, module, error)) - # Return the hash sum - if not output: - raise rpkgError('Could not find remote branch %s for %s' - % (branch, module)) - return output.split()[0] - - def gitbuildhash(self, build): - """Determine the git hash used to produce a particular N-V-R""" - - # Get the build data from the nvr - self.log.debug('Getting task data from the build system') - bdata = self.anon_kojisession.getBuild(build) - if not bdata: - raise rpkgError('Unknown build: %s' % build) - - # Get the task data out of that build data - taskinfo = self.anon_kojisession.getTaskRequest(bdata['task_id']) - # taskinfo is a list of items, first item is the task url. - # second is the build target. - # See if the build target starts with cvs or git - hash = None - buildsource = taskinfo[0] - if buildsource.startswith('cvs://'): - # snag everything after the last # mark - cvstag = buildsource.rsplit('#')[-1] - # Now read the remote repo to figure out the hash from the tag - giturl = self._get_namespace_anongiturl(bdata['name']) - cmd = ['git', 'ls-remote', '--tags', giturl, cvstag] - self.log.debug('Querying git server for tag info') - try: - output = subprocess.check_output(cmd) - hash = output.split()[0] - except: - # don't do anything here, we'll handle not having hash - # later - pass - elif buildsource.startswith('git://'): - # Match a 40 char block of text on the url line, that'll be - # our hash - hash = buildsource.rsplit('#')[-1] - else: - # Unknown build source - raise rpkgError('Unhandled build source %s' % buildsource) - if not hash: - raise rpkgError('Could not find hash of build %s' % build) - return (hash) - - def import_srpm(self, srpm): - """Import the contents of an srpm into a repo. - - srpm: File to import contents from - - This function will add/remove content to match the srpm, - - upload new files to the lookaside, and stage the changes. - - Returns a list of files to upload. - - """ - - # see if the srpm even exists - srpm = os.path.abspath(srpm) - if not os.path.exists(srpm): - raise rpkgError('File not found.') - # bail if we're dirty - if self.repo.is_dirty(): - raise rpkgError('There are uncommitted changes in your repo') - # Get the details of the srpm - name, files, uploadfiles = self._srpmdetails(srpm) - - # Need a way to make sure the srpm name matches the repo some how. - - # Get a list of files we're currently tracking - ourfiles = self.repo.git.ls_files().split('\n') - if ourfiles == ['']: - # Repository doesn't contain any files - ourfiles = [] - else: - # Trim out sources and .gitignore - for file in ('.gitignore', 'sources'): - try: - ourfiles.remove(file) - except ValueError: - pass - - # Things work better if we're in our module directory - oldpath = os.getcwd() - os.chdir(self.path) - - # Look through our files and if it isn't in the new files, remove it. - for file in ourfiles: - if file not in files: - self.log.info("Removing no longer used file: %s", file) - self.repo.index.remove([file]) - os.remove(file) - - # Extract new files - cmd = ['rpm2cpio', srpm] - # We have to force cpio to copy out (u) because git messes with - # timestamps - cmd2 = ['cpio', '-iud', '--quiet'] - - rpmcall = subprocess.Popen(cmd, stdout=subprocess.PIPE) - cpiocall = subprocess.Popen(cmd2, stdin=rpmcall.stdout) - output, err = cpiocall.communicate() - if output: - self.log.debug(output) - if err: - os.chdir(oldpath) - raise rpkgError("Got an error from rpm2cpio: %s" % err) - - # And finally add all the files we know about (and our stock files) - for file in ('.gitignore', 'sources'): - if not os.path.exists(file): - # Create the file - open(file, 'w').close() - files.append(file) - self.repo.index.add(files) - # Return to the caller and let them take it from there. - os.chdir(oldpath) - return(uploadfiles) - - def list_tag(self, tagname='*'): - """List all tags in the repository which match a given tagname. - - The optional `tagname` argument may be a shell glob (it is matched - with fnmatch). - - """ - if tagname is None: - tagname = '*' - - tags = map(lambda t: t.name, self.repo.tags) - - if tagname != '*': - tags = filter(lambda t: fnmatch.fnmatch(t, tagname), tags) - - for tag in tags: - print(tag) - - def new(self): - """Return changes in a repo since the last tag""" - - # Find the latest tag - try: - tag = self.repo.git.describe('--tags', '--abbrev=0') - except git.exc.GitCommandError: - raise rpkgError('Cannot get changes because there are no tags in this repo.') - # Now get the diff - self.log.debug('Diffing from tag %s', tag) - return self.repo.git.diff('-M', tag) - - def patch(self, suffix, rediff=False): - """Generate a patch from the expanded source and add it to index - - suffix: Look for files named with this suffix to diff - rediff: optionally retain any comments in the patch file and rediff - - Will create a patch file named name-version-suffix.patch - """ - - # Create the outfile name based on arguments - outfile = '%s-%s-%s.patch' % (self.module_name, self.ver, suffix) - - # If we want to rediff, the patch file has to already exist - if rediff and not os.path.exists(os.path.join(self.path, outfile)): - raise rpkgError('Patch file %s not found, unable to rediff' % - os.path.join(self.path, outfile)) - - # See if there is a source dir to diff in - if not os.path.isdir(os.path.join(self.path, - '%s-%s' % (self.module_name, - self.ver))): - raise rpkgError('Expanded source dir not found!') - - # Setup the command - cmd = ['gendiff', '%s-%s' % (self.module_name, self.ver), - '.%s' % suffix] - - # Try to run the command and capture the output - try: - self.log.debug('Running %s', ' '.join(cmd)) - (output, errors) = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=self.path).communicate() - except Exception as e: - raise rpkgError('Error running gendiff: %s' % e) - - # log any errors - if errors: - self.log.error(errors) - - # See if we got anything - if not output: - raise rpkgError('gendiff generated an empty patch!') - - # See if we are rediffing and handle the old patch file - if rediff: - oldpatch = open(os.path.join(self.path, outfile), 'r').readlines() - # back up the old file - self.log.debug('Moving existing patch %s to %s~', outfile, outfile) - os.rename(os.path.join(self.path, outfile), - '%s~' % os.path.join(self.path, outfile)) - # Capture the lines preceding the diff - newhead = [] - for line in oldpatch: - if line.startswith('diff'): - break - else: - newhead.append(line) - - log.debug('Saved from previous patch: \n%s' % ''.join(newhead)) - # Stuff the new head in front of the existing output - output = ''.join(newhead) + output - - # Write out the patch - open(os.path.join(self.path, outfile), 'w').write(output) - - # Add it to the index - # Again this returns a blank line we want to keep quiet - self.repo.index.add([outfile]) - log.info('Created %s and added it to the index' % outfile) - - def pull(self, rebase=False, norebase=False): - """Pull changes from the remote repository - - Optionally rebase current branch on top of remote branch - - Optionally override .git setting to always rebase - - """ - - cmd = ['git', 'pull'] - if self.quiet: - cmd.append('-q') - if rebase: - cmd.append('--rebase') - if norebase: - cmd.append('--no-rebase') - self._run_command(cmd, cwd=self.path) - return - - def find_untracked_patches(self): - """Find patches that are not tracked by git and sources both""" - file_pattern = os.path.join(self.path, '*.patch') - patches_in_repo = [os.path.basename(filename) for filename - in glob.glob(file_pattern)] - - git_tree = self.repo.head.commit.tree - sources_file = SourcesFile(self.sources_filename, - self.source_entry_type) - - patches_not_tracked = [ - patch for patch in patches_in_repo - if patch not in git_tree and patch not in sources_file] - - return patches_not_tracked - - def push(self, force=False): - """Push changes to the remote repository""" - - # see if our branch is tracking anything - try: - self.load_branch_merge() - except: - self.log.warning('Current branch cannot be pushed anywhere!') - - untracked_patches = self.find_untracked_patches() - if untracked_patches: - self.log.warning( - 'Patches %s %s not tracked within either git or sources', - ', '.join(untracked_patches), - 'is' if len(untracked_patches) == 1 else 'are') - - cmd = ['git', 'push'] - if self.quiet: - cmd.append('-q') - self._run_command(cmd, cwd=self.path) - - def sources(self, outdir=None): - """Download source files""" - - if not os.path.exists(self.sources_filename): - self.log.info("sources file doesn't exist. Source files download skipped.") - return - - # Default to putting the files where the module is - if not outdir: - outdir = self.path - - sourcesf = SourcesFile(self.sources_filename, self.source_entry_type) - - for entry in sourcesf.entries: - outfile = os.path.join(outdir, entry.file) - self.lookasidecache.download( - self.module_name, entry.file, entry.hash, outfile, - hashtype=entry.hashtype, branch=self.branch_merge) - - def switch_branch(self, branch, fetch=True): - """Switch the working branch - - Will create a local branch if one doesn't already exist, - based on / - - Logs output and returns nothing. - """ - - # Currently this just grabs the first matching branch name from - # the first remote it finds. When multiple remotes are in play - # this needs to get smarter - - # See if the repo is dirty first - if self.repo.is_dirty(): - raise rpkgError('%s has uncommitted changes. Use git status ' - 'to see details' % self.path) - - # Get our list of branches - (locals, remotes) = self._list_branches(fetch) - - if branch not in locals: - # We need to create a branch - self.log.debug('No local branch found, creating a new one') - totrack = None - full_branch = '%s/%s' % (self.branch_remote, branch) - for remote in remotes: - if remote == full_branch: - totrack = remote - break - else: - raise rpkgError('Unknown remote branch %s' % full_branch) - try: - self.log.info(self.repo.git.checkout('-b', branch, '--track', totrack)) - except Exception as err: - # This needs to be finer grained I think... - raise rpkgError('Could not create branch %s: %s' - % (branch, err)) - else: - try: - self.repo.git.checkout(branch) - # The above should have no output, but stash it anyway - self.log.info("Switched to branch '%s'", branch) - except Exception as err: - # This needs to be finer grained I think... - raise rpkgError('Could not check out %s\n%s' % (branch, - err.stderr)) - return - - def check_repo(self, is_dirty=True, all_pushed=True): - if is_dirty: - if self.repo.is_dirty(): - raise rpkgError('%s has uncommitted changes. Use git status ' - 'to see details' % self.path) - if all_pushed: - branch = self.repo.active_branch - remote = self.repo.git.config('--get', 'branch.%s.remote' % branch) - merge = self.repo.git.config('--get', 'branch.%s.merge' % branch).replace('refs/heads', - remote) - if self.repo.git.rev_list('%s...%s' % (merge, branch)): - raise rpkgError('There are unpushed changes in your repo') - - def build(self, skip_tag=False, scratch=False, background=False, - url=None, chain=None, arches=None, sets=False, nvr_check=True): - """Initiate a build of the module. Available options are: - - skip_tag: Skip the tag action after the build - - scratch: Perform a scratch build - - background: Perform the build with a low priority - - url: A url to an uploaded srpm to build from - - chain: A chain build set - - arches: A set of arches to limit the scratch build for - - sets: A boolean to let us know whether or not the chain has sets - - nvr_check: A boolean; locally construct NVR and submit a build only if - NVR doesn't exist in a build system - - This function submits the task to koji and returns the taskID - - It is up to the client to wait or watch the task. - """ - - # Ensure the repo exists as well as repo data and site data - # build up the command that a user would issue - cmd = [self.build_client] - # construct the url - if not url: - # We don't have a url, so build from the latest commit - # Check to see if the tree is dirty and if all local commits - # are pushed - self.check_repo() - url = self._get_namespace_anongiturl(self.ns_module_name) + \ - '?#%s' % self.commithash - # Check to see if the target is valid - build_target = self.kojisession.getBuildTarget(self.target) - if not build_target: - raise rpkgError('Unknown build target: %s' % self.target) - # see if the dest tag is locked - dest_tag = self.kojisession.getTag(build_target['dest_tag_name']) - if not dest_tag: - raise rpkgError('Unknown destination tag %s' - % build_target['dest_tag_name']) - if dest_tag['locked'] and not scratch: - raise rpkgError('Destination tag %s is locked' % dest_tag['name']) - # If we're chain building, make sure inheritance works - if chain: - cmd.append('chain-build') - ancestors = self.kojisession.getFullInheritance( - build_target['build_tag']) - ancestors = [ancestor['parent_id'] for ancestor in ancestors] - if dest_tag['id'] not in [build_target['build_tag']] + ancestors: - raise rpkgError('Packages in destination tag ' - '%(dest_tag_name)s are not inherited by' - 'build tag %(build_tag_name)s' % - build_target) - else: - cmd.append('build') - # define our dictionary for options - opts = {} - # Set a placeholder for the build priority - priority = None - if skip_tag: - opts['skip_tag'] = True - cmd.append('--skip-tag') - if scratch: - opts['scratch'] = True - cmd.append('--scratch') - if background: - cmd.append('--background') - priority = 5 # magic koji number :/ - if arches: - if not scratch: - raise rpkgError('Cannot override arches for non-scratch ' - 'builds') - for arch in arches: - if not re.match(r'^[0-9a-zA-Z_.]+$', arch): - raise rpkgError('Invalid architecture name: %s' % arch) - cmd.append('--arch-override=%s' % ','.join(arches)) - opts['arch_override'] = ' '.join(arches) - - cmd.append(self.target) - - if url.endswith('.src.rpm'): - srpm = os.path.basename(url) - build_reference = srpm - else: - try: - build_reference = self.nvr - except rpkgError as error: - self.log.warning(error) - if nvr_check: - self.log.info('Note: You can skip NVR construction & NVR' - ' check with --skip-nvr-check. See help for' - ' more info.') - raise rpkgError('Cannot continue without properly constructed NVR.') - else: - self.log.info('NVR checking will be skipped so I do not' - ' care that I am not able to construct NVR.' - ' I will refer this build by package name' - ' in following messages.') - build_reference = self.module_name - - # see if this build has been done. Does not check builds within - # a chain - if nvr_check and not scratch and not url.endswith('.src.rpm'): - build = self.kojisession.getBuild(self.nvr) - if build: - if build['state'] == 1: - raise rpkgError('Package %s has already been built\n' - 'Note: You can skip this check with' - ' --skip-nvr-check. See help for more' - ' info.' % self.nvr) - # Now submit the task and get the task_id to return - # Handle the chain build version - if chain: - self.log.debug('Adding %s to the chain', url) - # If we're dealing with build sets the behaviour of the last - # package changes, and we add it to the last (potentially empty) - # set. Otherwise the last package just gets added to the end of - # the chain. - if sets: - chain[-1].append(url) - else: - chain.append([url]) - # This next list comp is ugly, but it's how we properly get a : - # put in between each build set - cmd.extend(' : '.join([' '.join(build_sets) for build_sets in chain]).split()) - self.log.info('Chain building %s + %s for %s', build_reference, chain[:-1], self.target) - self.log.debug('Building chain %s for %s with options %s and a priority of %s', - chain, self.target, opts, priority) - self.log.debug(' '.join(cmd)) - task_id = self.kojisession.chainBuild(chain, self.target, opts, priority=priority) - # Now handle the normal build - else: - cmd.append(url) - self.log.info('Building %s for %s', build_reference, self.target) - self.log.debug('Building %s for %s with options %s and a priority of %s', - url, self.target, opts, priority) - self.log.debug(' '.join(cmd)) - task_id = self.kojisession.build(url, self.target, opts, priority=priority) - self.log.info('Created task: %s', task_id) - self.log.info('Task info: %s/taskinfo?taskID=%s', self.kojiweburl, task_id) - return task_id - - def clog(self, raw=False): - """Write the latest spec changelog entry to a clog file""" - - # This is a little ugly. We want to find where %changelog starts, - # then only deal with the content up to the first empty newline. - # Then remove any lines that start with $ or %, and then replace - # %% with % - - cloglines = [] - first = True - spec = open(os.path.join(self.path, self.spec), 'r').readlines() - for line in spec: - if line.lower().startswith('%changelog'): - # Grab all the lines below changelog - for line2 in spec[spec.index(line):]: - if line2.startswith('\n'): - break - if line2.startswith('$'): - continue - if line2.startswith('%'): - continue - if line2.startswith('*'): - if first: - # skip the email n/v/r line. Redundant - continue - # Otherwise what follows is the next entry - break - if first: - if not raw: - cloglines.append(line2.lstrip('- ').replace('%%', - '%')) - cloglines.append("\n") - else: - cloglines.append(line2.replace('%%', '%')) - first = False - else: - cloglines.append(line2.replace('%%', '%')) - - # Now open the clog file and write out the lines - clogfile = open(os.path.join(self.path, 'clog'), 'w') - clogfile.writelines(cloglines) - - def compile(self, arch=None, short=False, builddir=None, nocheck=False): - """Run rpmbuild -bc on a module - - optionally for a specific arch, or short-circuit it, or - define an alternate builddir - - Logs the output and returns nothing - """ - - # Get the sources - self.sources() - # setup the rpm command - cmd = ['rpmbuild'] - if builddir: - # Tack on a new builddir to the end of the defines - self.rpmdefines.append("--define '_builddir %s'" % - os.path.abspath(builddir)) - cmd.extend(self.rpmdefines) - if arch: - cmd.extend(['--target', arch]) - if short: - cmd.append('--short-circuit') - if nocheck: - cmd.append('--nocheck') - if self.quiet: - cmd.append('--quiet') - cmd.extend(['-bc', os.path.join(self.path, self.spec)]) - # Run the command - self._run_command(cmd, shell=True) - - def giturl(self): - """Return the git url that would be used for building""" - - url = self._get_namespace_anongiturl(self.ns_module_name) + \ - '?#%s' % self.commithash - return url - - def koji_upload(self, file, path, callback=None): - """Upload a file to koji - - file is the file you wish to upload - - path is the relative path on the server to upload to - - callback is the progress callback to use, if any - - Returns nothing or raises - """ - - # See if we actually have a file - if not os.path.exists(file): - raise rpkgError('No such file: %s' % file) - if not self.kojisession: - raise rpkgError('No active %s session.' % - os.path.basename(self.build_client)) - # This should have a try and catch koji errors - self.kojisession.uploadWrapper(file, path, callback=callback) - - def install(self, arch=None, short=False, builddir=None, nocheck=False): - """Run rpm -bi on a module - - optionally for a specific arch, short-circuit it, or - define an alternative builddir - - Logs the output and returns nothing - """ - - # Get the sources - self.sources() - # setup the rpm command - cmd = ['rpmbuild'] - if builddir: - # Tack on a new builddir to the end of the defines - self.rpmdefines.append("--define '_builddir %s'" % - os.path.abspath(builddir)) - cmd.extend(self.rpmdefines) - if arch: - cmd.extend(['--target', arch]) - if short: - cmd.append('--short-circuit') - if nocheck: - cmd.append('--nocheck') - if self.quiet: - cmd.append('--quiet') - cmd.extend(['-bi', os.path.join(self.path, self.spec)]) - # Run the command - self._run_command(cmd, shell=True) - return - - def lint(self, info=False, rpmlintconf=None): - """Run rpmlint over a built srpm - - Log the output and returns nothing - rpmlintconf is the name of the config file passed to rpmlint if - specified by the command line argument. - """ - - # Check for srpm - srpm = "%s-%s-%s.src.rpm" % (self.module_name, self.ver, self.rel) - if not os.path.exists(os.path.join(self.path, srpm)): - log.warning('No srpm found') - - # Get the possible built arches - arches = self._get_build_arches_from_spec() - rpms = [] - for arch in arches: - if os.path.exists(os.path.join(self.path, arch)): - # For each available arch folder, lists file and keep - # those ending with .rpm - rpms.extend([os.path.join(self.path, arch, file) - for file in os.listdir(os.path.join(self.path, - arch)) - if file.endswith('.rpm')]) - if not rpms: - log.warning('No rpm found') - cmd = ['rpmlint'] - if info: - cmd.extend(['-i']) - if rpmlintconf: - cmd.extend(["-f", os.path.join(self.path, rpmlintconf)]) - elif os.path.exists(os.path.join(self.path, ".rpmlint")): - cmd.extend(["-f", os.path.join(self.path, ".rpmlint")]) - cmd.append(os.path.join(self.path, self.spec)) - if os.path.exists(os.path.join(self.path, srpm)): - cmd.append(os.path.join(self.path, srpm)) - cmd.extend(rpms) - # Run the command - self._run_command(cmd, shell=True) - - def local(self, arch=None, hashtype=None, builddir=None): - """rpmbuild locally for given arch. - - Takes arch to build for, and hashtype to build with. - - Writes output to a log file and logs it to the logger - - Returns the returncode from the build call - """ - - # This could really use a list of arches to build for and loop over - # Get the sources - self.sources() - # build up the rpm command - cmd = ['rpmbuild'] - if builddir: - # Tack on a new builddir to the end of the defines - self.rpmdefines.append("--define '_builddir %s'" % - os.path.abspath(builddir)) - cmd.extend(self.rpmdefines) - # Figure out the hash type to use - if not hashtype: - # Try to determine the dist - hashtype = self._guess_hashtype() - # This may need to get updated if we ever change our checksum default - if not hashtype == 'sha256': - cmd.extend(["--define '_source_filedigest_algorithm %s'" - % hashtype, - "--define '_binary_filedigest_algorithm %s'" - % hashtype]) - if arch: - cmd.extend(['--target', arch]) - if self.quiet: - cmd.append('--quiet') - cmd.extend(['-ba', os.path.join(self.path, self.spec)]) - logfile = '.build-%s-%s.log' % (self.ver, self.rel) - # Run the command - self._run_command(cmd, shell=True, pipe=['tee', logfile]) - - # Not to be confused with mockconfig the property - def mock_config(self, target=None, arch=None): - """Generate a mock config based on branch data. - - Can use option target and arch to override autodiscovery. - Will return the mock config file text. - """ - - # Figure out some things about ourself. - if not target: - target = self.target - if not arch: - arch = self.localarch - - # Figure out if we have a valid build target - build_target = self.anon_kojisession.getBuildTarget(target) - if not build_target: - raise rpkgError('Unknown build target: %s\n' - 'Consider using the --target option' % target) - - try: - repoid = self.anon_kojisession.getRepo( - build_target['build_tag_name'])['id'] - except Exception: - raise rpkgError('Could not find a valid build repo') - - # Generate the config - config = koji.genMockConfig('%s-%s' % (target, arch), arch, - distribution=self.disttag, - tag_name=build_target['build_tag_name'], - repoid=repoid, - topurl=self.topurl) - - # Return the mess - return(config) - - def _config_dir_other(self, config_dir, filenames=('site-defaults.cfg', - 'logging.ini')): - """Populates mock config directory with other necessary files - - If files are found in system config directory for mock they are copied - to mock config directory defined as method's argument. Otherwise empty - files are created.""" - for filename in filenames: - system_filename = '/etc/mock/%s' % filename - tmp_filename = os.path.join(config_dir, filename) - if os.path.exists(system_filename): - try: - shutil.copy2(system_filename, tmp_filename) - except Exception as error: - raise rpkgError('Failed to create copy system config file' - ' %s: %s' % (filename, error)) - else: - try: - open(tmp_filename, 'w').close() - except Exception as error: - raise rpkgError('Failed to create empty mock config' - ' file %s: %s' - % (tmp_filename, error)) - - def _config_dir_basic(self, config_dir=None, root=None): - """Setup directory with essential mock config - - If config directory doesn't exist it will be created. If temporary - directory was created by this method and error occours during - processing, temporary directory is removed. Otherwise it caller's - responsibility to remove this directory. - - Returns used config directory""" - if not root: - root = self.mockconfig - if not config_dir: - my_config_dir = tempfile.mkdtemp(prefix="%s." % root, - suffix='mockconfig') - config_dir = my_config_dir - self.log.debug('New mock config directory: %s', config_dir) - else: - my_config_dir = None - - try: - config_content = self.mock_config() - except rpkgError as error: - self._cleanup_tmp_dir(my_config_dir) - raise rpkgError('Could not generate config file: %s' - % error) - - config_file = os.path.join(config_dir, '%s.cfg' % root) - try: - open(config_file, 'wb').write(config_content) - except IOError as error: - self._cleanup_tmp_dir(my_config_dir) - raise rpkgError('Could not write config file: %s' % error) - - return config_dir - - def _cleanup_tmp_dir(self, tmp_dir): - """Tries to remove directory and ignores EEXIST error - - If occoured directory not exist error (EEXIST) it silently continue. - Otherwise raise rpkgError exception.""" - if not tmp_dir: - return - try: - shutil.rmtree(tmp_dir) - except OSError as error: - if error.errno != errno.EEXIST: - raise rpkgError('Failed to remove temporary directory' - ' %s. Reason: %s.' % (tmp_dir, error)) - - def mockbuild(self, mockargs=[], root=None, hashtype=None): - """Build the package in mock, using mockargs - - Log the output and returns nothing - """ - - # Make sure we have an srpm to run on - self.srpm(hashtype=hashtype) - - # setup the command - cmd = ['mock'] - cmd.extend(mockargs) - if self.quiet: - cmd.append('--quiet') - - config_dir = None - if not root: - root = self.mockconfig - chroot_cfg = '/etc/mock/%s.cfg' % root - if not os.path.exists(chroot_cfg): - self.log.debug('Mock config %s was not found. Going to' - ' request koji to create new one.', chroot_cfg) - try: - config_dir = self._config_dir_basic(root=root) - except rpkgError as error: - raise rpkgError('Failed to create mock config directory:' - ' %s' % error) - self.log.debug('Temporary mock config directory: %s', config_dir) - try: - self._config_dir_other(config_dir) - except rpkgError as error: - self._cleanup_tmp_dir(config_dir) - raise rpkgError('Failed to populate mock config directory:' - ' %s' % error) - cmd.extend(['--configdir', config_dir]) - - cmd.extend(['-r', root, '--resultdir', self.mock_results_dir, - '--rebuild', self.srpmname]) - # Run the command - try: - self._run_command(cmd) - finally: - self.log.debug('Cleaning up mock temporary config directory: %s', config_dir) - self._cleanup_tmp_dir(config_dir) - - def upload(self, files, replace=False): - """Upload source file(s) in the lookaside cache - - Can optionally replace the existing tracked sources - """ - - sourcesf = SourcesFile(self.sources_filename, self.source_entry_type, - replace=replace) - gitignore = GitIgnore(os.path.join(self.path, '.gitignore')) - - for f in files: - # TODO: Skip empty file needed? - file_hash = self.lookasidecache.hash_file(f) - file_basename = os.path.basename(f) - - try: - sourcesf.add_entry(self.lookasidehash, file_basename, - file_hash) - except HashtypeMixingError as e: - msg = '\n'.join([ - 'Can not upload a new source file with a %(newhash)s ' - 'hash, as the "%(sources)s" file contains at least one ' - 'line with a %(existinghash)s hash.', '', - 'Please redo the whole "%(sources)s" file using:', - ' `%(arg0)s new-sources file1 file2 ...`']) % { - 'newhash': e.new_hashtype, - 'existinghash': e.existing_hashtype, - 'sources': self.sources_filename, - 'arg0': sys.argv[0], - } - raise rpkgError(msg) - - gitignore.add('/%s' % file_basename) - self.lookasidecache.upload(self.module_name, f, file_hash) - - sourcesf.write() - gitignore.write() - - self.repo.index.add(['sources', '.gitignore']) - - def prep(self, arch=None, builddir=None): - """Run rpm -bp on a module - - optionally for a specific arch, or - define an alternative builddir - - Logs the output and returns nothing - """ - - # Get the sources - self.sources() - # setup the rpm command - cmd = ['rpmbuild'] - if builddir: - # Tack on a new builddir to the end of the defines - self.rpmdefines.append("--define '_builddir %s'" % - os.path.abspath(builddir)) - cmd.extend(self.rpmdefines) - if arch: - cmd.extend(['--target', arch]) - if self.quiet: - cmd.append('--quiet') - cmd.extend(['--nodeps', '-bp', os.path.join(self.path, self.spec)]) - # Run the command - self._run_command(cmd, shell=True) - - def srpm(self, hashtype=None): - """Create an srpm using hashtype from content in the module - - Requires sources already downloaded. - """ - - self.srpmname = os.path.join(self.path, - "%s-%s-%s.src.rpm" - % (self.module_name, self.ver, self.rel)) - - # See if we need to build the srpm - if os.path.exists(self.srpmname): - self.log.debug('Srpm found, rewriting it.') - - cmd = ['rpmbuild'] - cmd.extend(self.rpmdefines) - if self.quiet: - cmd.append('--quiet') - # Figure out which hashtype to use, if not provided one - if not hashtype: - # Try to determine the dist - hashtype = self._guess_hashtype() - # This may need to get updated if we ever change our checksum default - if not hashtype == 'sha256': - cmd.extend(["--define '_source_filedigest_algorithm %s'" - % hashtype, - "--define '_binary_filedigest_algorithm %s'" - % hashtype]) - cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)]) - self._run_command(cmd, shell=True) - - def unused_patches(self): - """Discover patches checked into source control that are not used - - Returns a list of unused patches, which may be empty. - """ - - # Create a list for unused patches - unused = [] - # Get the content of spec into memory for fast searching - with open(self.spec, 'r') as f: - data = f.read() - try: - spec = data.decode('UTF-8') - except UnicodeDecodeError as error: - # when can't decode file, ignore chars and show warning - spec = data.decode('UTF-8', 'ignore') - line, offset = self._byte_offset_to_line_number(spec, error.start) - self.log.warning("'%s' codec can't decode byte in position %d:%d : %s", - error.encoding, line, offset, error.reason) - # Replace %{name} with the package name - spec = spec.replace("%{name}", self.module_name) - # Replace %{version} with the package version - spec = spec.replace("%{version}", self.ver) - - # Get a list of files tracked in source control - files = self.repo.git.ls_files('--exclude-standard').split() - for file in files: - # throw out non patches - if not file.endswith(('.patch', '.diff')): - continue - if file not in spec: - unused.append(file) - return unused - - def _byte_offset_to_line_number(self, text, offset): - """ - Convert byte offset (given by e.g. DecodeError) to human readable - format (line number and char position) - Return a list with line number and char offset - """ - offset_inc = 0 - line_num = 1 - for line in text.split('\n'): - if offset_inc + len(line) + 1 > offset: - break - else: - offset_inc += len(line) + 1 - line_num += 1 - return [line_num, offset - offset_inc + 1] - - def verify_files(self, builddir=None): - """Run rpmbuild -bl on a module to verify the %files section - - optionally define an alternate builddir - """ - - # setup the rpm command - cmd = ['rpmbuild'] - if builddir: - # Tack on a new builddir to the end of the defines - self.rpmdefines.append("--define '_builddir %s'" % - os.path.abspath(builddir)) - cmd.extend(self.rpmdefines) - if self.quiet: - cmd.append('--quiet') - cmd.extend(['-bl', os.path.join(self.path, self.spec)]) - # Run the command - self._run_command(cmd, shell=True) - - def osbs_build(self, config_file, config_section, target_override=False, - yum_repourls=[], nowait=False): - self.check_repo() - os_conf = Configuration(conf_file=config_file, conf_section=config_section) - build_conf = Configuration(conf_file=config_file, conf_section=config_section) - osbs = OSBS(os_conf, build_conf) - - git_uri = re.sub(r"^git\+ssh", "git", self.push_url) - git_uri = re.sub("^ssh", "git", git_uri) - git_uri = re.sub("[^/]+@", "", git_uri) - git_ref = self.commithash - git_branch = self.branch_merge - user = self.user - component = self.module_name - docker_target = self.target - if not target_override: - # Translate the build target into a docker target, - # but only if --target wasn't specified on the command-line - docker_target = '%s-docker-candidate' % self.target.split('-candidate')[0] - - build = osbs.create_build( - git_uri=git_uri, - git_ref=git_ref, - git_branch=git_branch, - user=user, - component=component, - target=docker_target, - architecture="x86_64", - yum_repourls=yum_repourls - ) - build_id = build.build_id - - if nowait: - self.log.info('Build submitted: %s', build_id) - return - - print("Build submitted (%s), watching logs (feel free to interrupt)" % build_id) - for line in osbs.get_build_logs(build_id, follow=True): - print(line) - build_response = osbs.wait_for_build_to_finish(build_id) - if build_response.is_succeeded(): - repositories = build_response.get_repositories() - if repositories: - image_names = repositories.get("primary", []) + repositories.get("unique", []) - print("You can pull the image with one of the following commands:") - for image in image_names: - print(" docker pull %s" % image) - else: - raise RuntimeError( - "Build '%s' wasn't processed correctly. Please, report this." % build_id) - else: - raise RuntimeError("Build has failed.") - - def container_build_koji(self, target_override=False, opts={}, - kojiconfig=None, build_client=None, - koji_task_watcher=None, - nowait=False): - # check if repo is dirty and all commits are pushed - self.check_repo() - docker_target = self.target - if not target_override: - # Translate the build target into a docker target, - # but only if --target wasn't specified on the command-line - docker_target = '%s-docker-candidate' % self.target.split('-candidate')[0] - - koji_session_backup = (self.build_client, self.kojiconfig) - (self.build_client, self.kojiconfig) = (build_client, kojiconfig) - try: - self.load_kojisession() - if "buildContainer" not in self.kojisession.system.listMethods(): - raise RuntimeError("Kojihub instance does not support buildContainer") - - build_target = self.kojisession.getBuildTarget(docker_target) - if not build_target: - msg = "Unknown build target: %s" % docker_target - self.log.error(msg) - raise UnknownTargetError(msg) - else: - dest_tag = self.kojisession.getTag(build_target['dest_tag']) - if not dest_tag: - self.log.error("Unknown destination tag: %s", build_target['dest_tag_name']) - if dest_tag['locked'] and 'scratch' not in opts: - self.log.error("Destination tag %s is locked", dest_tag['name']) - - source = self._get_namespace_anongiturl(self.ns_module_name) - source += "#%s" % self.commithash - - task_opts = {} - for key in ('scratch', 'name', 'version', 'release', - 'yum_repourls', 'git_branch'): - if key in opts: - task_opts[key] = opts[key] - priority = opts.get("priority", None) - task_id = self.kojisession.buildContainer(source, - docker_target, - task_opts, - priority=priority) - self.log.info('Created task: %s', task_id) - self.log.info('Task info: %s/taskinfo?taskID=%s', self.kojiweburl, task_id) - if not nowait: - rv = koji_task_watcher(self.kojisession, [task_id]) - if rv == 0: - result = self.kojisession.getTaskResult(task_id) - try: - result["koji_builds"] = [ - "%s/buildinfo?buildID=%s" % (self.kojiweburl, - build_id) - for build_id in result.get("koji_builds", [])] - except TypeError: - pass - log_result(self.log.info, result) - - finally: - (self.build_client, self.kojiconfig) = koji_session_backup - self.load_kojisession() - - def container_build_setup(self, get_autorebuild=None, - set_autorebuild=None): - cfp = ConfigParser.SafeConfigParser() - if os.path.exists(self.osbs_config_filename): - cfp.read(self.osbs_config_filename) - - if get_autorebuild is not None: - if not cfp.has_option('autorebuild', 'enabled'): - self.log.info('true') - else: - self.log.info('true' if cfp.getboolean('autorebuild', 'enabled') else 'false') - elif set_autorebuild is not None: - if not cfp.has_section('autorebuild'): - cfp.add_section('autorebuild') - - cfp.set('autorebuild', 'enabled', set_autorebuild) - with open(self.osbs_config_filename, 'w') as fp: - cfp.write(fp) - - self.repo.index.add([self.osbs_config_filename]) - self.log.info("Config value changed, don't forget to commit %s file", - self.osbs_config_filename) - else: - self.log.info('Nothing to be done') - - def copr_build(self, project, srpm_name, nowait): - cmd = ['copr-cli', 'build'] - if nowait: - cmd.append('--nowait') - cmd.extend([project, srpm_name]) - self._run_command(cmd) diff --git a/src/pyrpkg/cli.py b/src/pyrpkg/cli.py deleted file mode 100755 index a48d7c9..0000000 --- a/src/pyrpkg/cli.py +++ /dev/null @@ -1,1607 +0,0 @@ -# cli.py - a cli client class module -# -# Copyright (C) 2011 Red Hat Inc. -# Author(s): Jesse Keating -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. -# -# There are 6 functions derived from /usr/bin/koji which are licensed under -# LGPLv2.1. See comments before those functions. - -import argparse -import sys -import os -import logging -import time -import random -import string -from six.moves import xmlrpc_client -import pwd -import koji - -import utils - -OSBS_DEFAULT_CONF_FILE = "/etc/osbs/osbs.conf" - - -class cliClient(object): - """This is a client class for rpkg clients.""" - - def __init__(self, config, name=None): - """This requires a ConfigParser object - - Name of the app can optionally set, or discovered from exe name - """ - - self.config = config - self._name = name - # Define default name in child class - # self.DEFAULT_CLI_NAME = None - # Property holders, set to none - self._cmd = None - self._module = None - # Setup the base argparser - self.setup_argparser() - # Add a subparser - self.subparsers = self.parser.add_subparsers( - title='Targets', - description='These are valid commands you can ask %s to do' - % self.name) - # Register all the commands - self.setup_subparsers() - - @property - def name(self): - """Property used to identify prog name and key in config file""" - - if not self._name: - self._name = self.get_name() - assert self._name - return(self._name) - - def get_name(self): - name = os.path.basename(sys.argv[0]) - if not name or '__main__.py' in name: - try: - name = self.DEFAULT_CLI_NAME - except AttributeError: - # Ignore missing DEFAULT_CLI_NAME for backwards - # compatibility - pass - if not name: - # We don't have logger available yet - sys.stderr.write('Could not determine CLI name\n') - sys.exit(1) - return name - - # Define some properties here, for lazy loading - @property - def cmd(self): - """This is a property for the command attribute""" - - if not self._cmd: - self.load_cmd() - return(self._cmd) - - def load_cmd(self): - """This sets up the cmd object""" - - # Set target if we got it as an option - target = None - if hasattr(self.args, 'target') and self.args.target: - target = self.args.target - - # load items from the config file - items = dict(self.config.items(self.name, raw=True)) - - dg_namespaced = items.get("distgit_namespaced", False) - - # Create the cmd object - self._cmd = self.site.Commands(self.args.path, - items['lookaside'], - items['lookasidehash'], - items['lookaside_cgi'], - items['gitbaseurl'], - items['anongiturl'], - items['branchre'], - items['kojiconfig'], - items['build_client'], - user=self.args.user, - dist=self.args.dist, - target=target, - quiet=self.args.q, - distgit_namespaced=dg_namespaced - ) - - self._cmd.module_name = self.args.module_name - self._cmd.password = self.args.password - self._cmd.runas = self.args.runas - self._cmd.debug = self.args.debug - self._cmd.verbose = self.args.v - self._cmd.clone_config = items.get('clone_config') - - # This function loads the extra stuff once we figure out what site - # we are - def do_imports(self, site=None): - """Import extra stuff not needed during build - - As a side effect method sets self.site with a loaded library. - - site option can be used to specify which library to load - """ - - # We do some imports here to be more flexible - if not site: - import pyrpkg - self.site = pyrpkg - else: - try: - __import__(site) - self.site = sys.modules[site] - except ImportError: - raise Exception('Unknown site %s' % site) - - def setup_argparser(self): - """Setup the argument parser and register some basic commands.""" - - self.parser = argparse.ArgumentParser( - prog=self.name, - epilog='For detailed help pass --help to a target') - # Add some basic arguments that should be used by all. - # Add a config file - self.parser.add_argument('--config', '-C', - default=None, - help='Specify a config file to use') - # Allow forcing the dist value - self.parser.add_argument('--dist', default=None, - help='Override the discovered distribution') - # Allow forcing the package name - self.parser.add_argument('--module-name', - help=('Override the module name. Otherwise' - ' it is discovered from: Git push URL' - ' or Git URL (last part of path with' - ' .git extension removed) or from name' - ' macro in spec file. In that order.') - ) - # Override the discovered user name - self.parser.add_argument('--user', default=None, - help='Override the discovered user name') - # If using password auth - self.parser.add_argument('--password', default=None, - help='Password for Koji login') - # Run Koji commands as a user other then the one you have - # credentials for (requires configuration on the Koji hub) - self.parser.add_argument('--runas', default=None, - help='Run Koji commands as a different user') - # Let the user define a path to work in rather than cwd - self.parser.add_argument('--path', default=None, - type=utils.u, - help='Define the directory to work in ' - '(defaults to cwd)') - # Verbosity - self.parser.add_argument('--verbose', '-v', dest='v', - action='store_true', - help='Run with verbose debug output') - self.parser.add_argument('--debug', '-d', dest='debug', - action='store_true', - help='Run with debug output') - self.parser.add_argument('-q', action='store_true', - help='Run quietly only displaying errors') - - def setup_subparsers(self): - """Setup basic subparsers that all clients should use""" - - # Setup some basic shared subparsers - - # help command - self.register_help() - - # Add a common parsers - self.register_build_common() - self.register_rpm_common() - - # Other targets - self.register_build() - self.register_chainbuild() - self.register_clean() - self.register_clog() - self.register_clone() - self.register_copr_build() - self.register_commit() - self.register_compile() - self.register_container_build() - self.register_container_build_setup() - self.register_diff() - self.register_gimmespec() - self.register_gitbuildhash() - self.register_giturl() - self.register_import_srpm() - self.register_install() - self.register_lint() - self.register_local() - self.register_mockbuild() - self.register_mock_config() - self.register_new() - self.register_new_sources() - self.register_patch() - self.register_prep() - self.register_pull() - self.register_push() - self.register_scratch_build() - self.register_sources() - self.register_srpm() - self.register_switch_branch() - self.register_tag() - self.register_unused_patches() - self.register_upload() - self.register_verify_files() - self.register_verrel() - - # All the register functions go here. - def register_help(self): - """Register the help command.""" - - help_parser = self.subparsers.add_parser('help', help='Show usage') - help_parser.set_defaults(command=self.parser.print_help) - - # Setup a couple common parsers to save code duplication - def register_build_common(self): - """Create a common build parser to use in other commands""" - - self.build_parser_common = argparse.ArgumentParser( - 'build_common', add_help=False) - self.build_parser_common.add_argument( - '--arches', nargs='*', help='Build for specific arches') - self.build_parser_common.add_argument( - '--md5', action='store_const', const='md5', default=None, - dest='hash', help='Use md5 checksums (for older rpm hosts)') - self.build_parser_common.add_argument( - '--nowait', action='store_true', default=False, - help="Don't wait on build") - self.build_parser_common.add_argument( - '--target', default=None, - help='Define build target to build into') - self.build_parser_common.add_argument( - '--background', action='store_true', default=False, - help='Run the build at a low priority') - - def register_rpm_common(self): - """Create a common parser for rpm commands""" - - self.rpm_parser_common = argparse.ArgumentParser( - 'rpm_common', add_help=False) - self.rpm_parser_common.add_argument( - '--builddir', default=None, help='Define an alternate builddir') - self.rpm_parser_common.add_argument( - '--arch', help='Prep for a specific arch') - - def register_build(self): - """Register the build target""" - - build_parser = self.subparsers.add_parser( - 'build', help='Request build', parents=[self.build_parser_common], - description='This command requests a build of the package in the ' - 'build system. By default it discovers the target ' - 'to build for based on branch data, and uses the ' - 'latest commit as the build source.') - build_parser.add_argument( - '--skip-nvr-check', action='store_false', default=True, - dest='nvr_check', - help='Submit build to buildsystem without check if NVR was ' - 'already build. NVR is constructed locally and may be ' - 'different from NVR constructed during build on builder.') - build_parser.add_argument( - '--skip-tag', action='store_true', default=False, - help='Do not attempt to tag package') - build_parser.add_argument( - '--scratch', action='store_true', default=False, - help='Perform a scratch build') - build_parser.add_argument( - '--srpm', nargs='?', const='CONSTRUCT', - help='Build from an srpm. If no srpm is provided with this option' - ' an srpm will be generated from current module content.') - build_parser.set_defaults(command=self.build) - - def register_chainbuild(self): - """Register the chain build target""" - - chainbuild_parser = self.subparsers.add_parser( - 'chain-build', parents=[self.build_parser_common], - help='Build current package in order with other packages', - formatter_class=argparse.RawDescriptionHelpFormatter, - description=""" -Build current package in order with other packages. - -example: %(name)s chain-build libwidget libgizmo - -The current package is added to the end of the CHAIN list. -Colons (:) can be used in the CHAIN parameter to define groups of -packages. Packages in any single group will be built in parallel -and all packages in a group must build successfully and populate -the repository before the next group will begin building. - -For example: - -%(name)s chain-build libwidget libaselib : libgizmo : - -will cause libwidget and libaselib to be built in parallel, followed -by libgizmo and then the current directory package. If no groups are -defined, packages will be built sequentially.""" % {'name': self.name}) - chainbuild_parser.add_argument( - 'package', nargs='+', - help='List the packages and order you want to build in') - chainbuild_parser.set_defaults(command=self.chainbuild) - - def register_clean(self): - """Register the clean target""" - clean_parser = self.subparsers.add_parser( - 'clean', help='Remove untracked files', - description="This command can be used to clean up your working " - "directory. By default it will follow .gitignore " - "rules.") - clean_parser.add_argument( - '--dry-run', '-n', action='store_true', help='Perform a dry-run') - clean_parser.add_argument( - '-x', action='store_true', help='Do not follow .gitignore rules') - clean_parser.set_defaults(command=self.clean) - - def register_clog(self): - """Register the clog target""" - - clog_parser = self.subparsers.add_parser( - 'clog', help='Make a clog file containing top changelog entry', - description='This will create a file named "clog" that contains ' - 'the latest rpm changelog entry. The leading "- " ' - 'text will be stripped.') - clog_parser.add_argument( - '--raw', action='store_true', default=False, - help='Generate a more "raw" clog without twiddling the contents') - clog_parser.set_defaults(command=self.clog) - - def register_clone(self): - """Register the clone target and co alias""" - - clone_parser = self.subparsers.add_parser( - 'clone', help='Clone and checkout a module', - description='This command will clone the named module from the ' - 'configured repository base URL. By default it will ' - 'also checkout the master branch for your working ' - 'copy.') - # Allow an old style clone with subdirs for branches - clone_parser.add_argument( - '--branches', '-B', action='store_true', - help='Do an old style checkout with subdirs for branches') - # provide a convenient way to get to a specific branch - clone_parser.add_argument( - '--branch', '-b', help='Check out a specific branch') - # allow to clone without needing a account on the scm server - clone_parser.add_argument( - '--anonymous', '-a', action='store_true', - help='Check out a module anonymously') - # store the module to be cloned - clone_parser.add_argument( - 'module', nargs=1, help='Name of the module to clone') - # Eventually specify where to clone the module - clone_parser.add_argument( - "clone_target", default=None, nargs="?", - help='Directory in which to clone the module') - clone_parser.set_defaults(command=self.clone) - - # Add an alias for historical reasons - co_parser = self.subparsers.add_parser( - 'co', parents=[clone_parser], conflict_handler='resolve', - help='Alias for clone') - co_parser.set_defaults(command=self.clone) - - def register_commit(self): - """Register the commit target and ci alias""" - - commit_parser = self.subparsers.add_parser( - 'commit', help='Commit changes', - description='This invokes a git commit. All tracked files with ' - 'changes will be committed unless a specific file ' - 'list is provided. $EDITOR will be used to generate a' - ' changelog message unless one is given to the ' - 'command. A push can be done at the same time.') - commit_parser.add_argument( - '-c', '--clog', default=False, action='store_true', - help='Generate the commit message from the Changelog section') - commit_parser.add_argument( - '--raw', action='store_true', default=False, - help='Make the clog raw') - commit_parser.add_argument( - '-t', '--tag', default=False, action='store_true', - help='Create a tag for this commit') - commit_parser.add_argument( - '-m', '--message', default=None, - help='Use the given as the commit message') - commit_parser.add_argument( - '-F', '--file', default=None, - help='Take the commit message from the given file') - # allow one to commit /and/ push at the same time. - commit_parser.add_argument( - '-p', '--push', default=False, action='store_true', - help='Commit and push as one action') - # Allow a list of files to be committed instead of everything - commit_parser.add_argument( - 'files', nargs='*', default=[], - help='Optional list of specific files to commit') - commit_parser.add_argument( - '-s', '--signoff', default=False, action='store_true', - help='Include a signed-off-by') - commit_parser.set_defaults(command=self.commit) - - # Add a ci alias - ci_parser = self.subparsers.add_parser( - 'ci', parents=[commit_parser], conflict_handler='resolve', - help='Alias for commit') - ci_parser.set_defaults(command=self.commit) - - def register_compile(self): - """Register the compile target""" - - compile_parser = self.subparsers.add_parser( - 'compile', parents=[self.rpm_parser_common], - help='Local test rpmbuild compile', - description='This command calls rpmbuild to compile the source. ' - 'By default the prep and configure stages will be ' - 'done as well, unless the short-circuit option is ' - 'used.') - compile_parser.add_argument('--short-circuit', - action='store_true', - help='short-circuit compile') - compile_parser.add_argument('--nocheck', - action='store_true', - help='nocheck compile') - compile_parser.set_defaults(command=self.compile) - - def register_diff(self): - """Register the diff target""" - - diff_parser = self.subparsers.add_parser( - 'diff', help='Show changes between commits, commit and working ' - 'tree, etc', - description='Use git diff to show changes that have been made to ' - 'tracked files. By default cached changes (changes ' - 'that have been git added) will not be shown.') - diff_parser.add_argument( - '--cached', default=False, action='store_true', - help='View staged changes') - diff_parser.add_argument( - 'files', nargs='*', default=[], - help='Optionally diff specific files') - diff_parser.set_defaults(command=self.diff) - - def register_gimmespec(self): - """Register the gimmespec target""" - - gimmespec_parser = self.subparsers.add_parser( - 'gimmespec', help='Print the spec file name') - gimmespec_parser.set_defaults(command=self.gimmespec) - - def register_gitbuildhash(self): - """Register the gitbuildhash target""" - - gitbuildhash_parser = self.subparsers.add_parser( - 'gitbuildhash', - help='Print the git hash used to build the provided n-v-r', - description='This will show you the commit hash string used to ' - 'build the provided build n-v-r') - gitbuildhash_parser.add_argument( - 'build', help='name-version-release of the build to query.') - gitbuildhash_parser.set_defaults(command=self.gitbuildhash) - - def register_giturl(self): - """Register the giturl target""" - - giturl_parser = self.subparsers.add_parser( - 'giturl', help='Print the git url for building', - description='This will show you which git URL would be used in a ' - 'build command. It uses the git hashsum of the HEAD ' - 'of the current branch (which may not be pushed).') - giturl_parser.set_defaults(command=self.giturl) - - def register_import_srpm(self): - """Register the import-srpm target""" - - import_srpm_parser = self.subparsers.add_parser( - 'import', help='Import srpm content into a module', - description='This will extract sources, patches, and the spec ' - 'file from an srpm and update the current module ' - 'accordingly. It will import to the current branch by ' - 'default.') - import_srpm_parser.add_argument( - '--skip-diffs', help="Don't show diffs when import srpms", - action='store_true') - import_srpm_parser.add_argument('srpm', help='Source rpm to import') - import_srpm_parser.set_defaults(command=self.import_srpm) - - def register_install(self): - """Register the install target""" - - install_parser = self.subparsers.add_parser( - 'install', parents=[self.rpm_parser_common], - help='Local test rpmbuild install', - description='This will call rpmbuild to run the install section. ' - 'All leading sections will be processed as well, ' - 'unless the short-circuit option is used.') - install_parser.add_argument( - '--short-circuit', - action='store_true', - default=False, - help='short-circuit install') - install_parser.add_argument( - '--nocheck', - action='store_true', - help='nocheck install') - install_parser.set_defaults(command=self.install, default=False) - - def register_lint(self): - """Register the lint target""" - - lint_parser = self.subparsers.add_parser( - 'lint', help='Run rpmlint against local spec and build output if ' - 'present.', - description='Rpmlint can be configured using the --rpmlintconf/-r' - ' option or by setting a .rpmlint file in the ' - 'working directory') - lint_parser.add_argument( - '--info', '-i', default=False, action='store_true', - help='Display explanations for reported messages') - lint_parser.add_argument( - '--rpmlintconf', '-r', default=None, - help='Use a specific configuration file for rpmlint') - lint_parser.set_defaults(command=self.lint) - - def register_local(self): - """Register the local target""" - - local_parser = self.subparsers.add_parser( - 'local', parents=[self.rpm_parser_common], - help='Local test rpmbuild binary', - description='Locally test run of rpmbuild producing binary RPMs. ' - 'The rpmbuild output will be logged into a file named' - ' .build-%{version}-%{release}.log') - # Allow the user to just pass "--md5" which will set md5 as the - # hash, otherwise use the default of sha256 - local_parser.add_argument( - '--md5', action='store_const', const='md5', default=None, - dest='hash', help='Use md5 checksums (for older rpm hosts)') - local_parser.set_defaults(command=self.local) - - def register_new(self): - """Register the new target""" - - new_parser = self.subparsers.add_parser( - 'new', help='Diff against last tag', - description='This will use git to show a diff of all the changes ' - '(even uncommitted changes) since the last git tag ' - 'was applied.') - new_parser.set_defaults(command=self.new) - - def register_mockbuild(self): - """Register the mockbuild target""" - - mockbuild_parser = self.subparsers.add_parser( - 'mockbuild', help='Local test build using mock', - description='This will use the mock utility to build the package ' - 'for the distribution detected from branch ' - 'information. This can be overridden using the global' - ' --dist option. Your user must be in the local ' - '"mock" group.', - epilog="If config file for mock isn't found in the " - "/etc/mock directory, a temporary config " - "directory for mock is created and populated " - "with a config file created with mock-config.") - mockbuild_parser.add_argument('--root', help='Override mock root') - # Allow the user to just pass "--md5" which will set md5 as the - # hash, otherwise use the default of sha256 - mockbuild_parser.add_argument( - '--md5', action='store_const', const='md5', default=None, - dest='hash', help='Use md5 checksums (for older rpm hosts)') - mockbuild_parser.add_argument( - '--no-clean', '-n', help='Do not clean chroot before building ' - 'package', action='store_true') - mockbuild_parser.add_argument( - '--no-cleanup-after', help='Do not clean chroot after building ' - '(if automatic cleanup is enabled', action='store_true') - mockbuild_parser.add_argument( - '--no-clean-all', '-N', help='Alias for both --no-clean and ' - '--no-cleanup-after', action='store_true') - mockbuild_parser.set_defaults(command=self.mockbuild) - - def register_mock_config(self): - """Register the mock-config target""" - - mock_config_parser = self.subparsers.add_parser( - 'mock-config', help='Generate a mock config', - description='This will generate a mock config based on the ' - 'buildsystem target') - mock_config_parser.add_argument( - '--target', help='Override target used for config', default=None) - mock_config_parser.add_argument('--arch', help='Override local arch') - mock_config_parser.set_defaults(command=self.mock_config) - - def register_new_sources(self): - """Register the new-sources target""" - - # Make it part of self to be used later - self.new_sources_parser = self.subparsers.add_parser( - 'new-sources', help='Upload new source files', - description='This will upload new source files to the lookaside ' - 'cache and remove any existing ones. The "sources" ' - 'and .gitignore files will be updated with the new ' - 'uploaded file(s).') - self.new_sources_parser.add_argument('files', nargs='+') - self.new_sources_parser.set_defaults( - command=self.new_sources, replace=True) - - def register_patch(self): - """Register the patch target""" - - patch_parser = self.subparsers.add_parser( - 'patch', help='Create and add a gendiff patch file', - epilog='Patch file will be named: package-version-suffix.patch ' - 'and the file will be added to the repo index') - patch_parser.add_argument( - '--rediff', action='store_true', default=False, - help='Recreate gendiff file retaining comments Saves old patch ' - 'file with a suffix of ~') - patch_parser.add_argument( - 'suffix', help='Look for files with this suffix to diff') - patch_parser.set_defaults(command=self.patch) - - def register_prep(self): - """Register the prep target""" - - prep_parser = self.subparsers.add_parser( - 'prep', parents=[self.rpm_parser_common], - help='Local test rpmbuild prep', - description='Use rpmbuild to "prep" the sources (unpack the ' - 'source archive(s) and apply any patches.)') - prep_parser.set_defaults(command=self.prep) - - def register_pull(self): - """Register the pull target""" - - pull_parser = self.subparsers.add_parser( - 'pull', help='Pull changes from the remote repository and update ' - 'the working copy.', - description='This command uses git to fetch remote changes and ' - 'apply them to the current working copy. A rebase ' - 'option is available which can be used to avoid ' - 'merges.', - epilog='See git pull --help for more details') - pull_parser.add_argument( - '--rebase', action='store_true', - help='Rebase the locally committed changes on top of the remote ' - 'changes after fetching. This can avoid a merge commit, but ' - 'does rewrite local history.') - pull_parser.add_argument( - '--no-rebase', action='store_true', - help='Do not rebase, overriding .git settings to the contrary') - pull_parser.set_defaults(command=self.pull) - - def register_push(self): - """Register the push target""" - - push_parser = self.subparsers.add_parser( - 'push', help='Push changes to remote repository') - push_parser.add_argument('--force', '-f', help='Force push', action='store_true') - push_parser.set_defaults(command=self.push) - - def register_scratch_build(self): - """Register the scratch-build target""" - - scratch_build_parser = self.subparsers.add_parser( - 'scratch-build', help='Request scratch build', - parents=[self.build_parser_common], - description='This command will request a scratch build of the ' - 'package. Without providing an srpm, it will attempt ' - 'to build the latest commit, which must have been ' - 'pushed. By default all appropriate arches will be ' - 'built.') - scratch_build_parser.add_argument( - '--srpm', nargs='?', const='CONSTRUCT', - help='Build from an srpm. If no srpm is provided with this ' - 'option an srpm will be generated from the current module ' - 'content.') - scratch_build_parser.set_defaults(command=self.scratch_build) - - def register_sources(self): - """Register the sources target""" - - sources_parser = self.subparsers.add_parser( - 'sources', help='Download source files') - sources_parser.add_argument( - '--outdir', default=os.curdir, - help='Directory to download files into (defaults to pwd)') - sources_parser.set_defaults(command=self.sources) - - def register_srpm(self): - """Register the srpm target""" - - srpm_parser = self.subparsers.add_parser( - 'srpm', help='Create a source rpm') - # optionally define old style hashsums - srpm_parser.add_argument( - '--md5', action='store_const', const='md5', default=None, - dest='hash', help='Use md5 checksums (for older rpm hosts)') - srpm_parser.set_defaults(command=self.srpm) - - def register_copr_build(self): - """Register the copr-build target""" - - copr_parser = self.subparsers.add_parser( - 'copr-build', help='Build package in Copr', - formatter_class=argparse.RawDescriptionHelpFormatter, - description=""" -Build package in Copr. - -Note: you need to have set up correct api key. For more information -see API KEY section of copr-cli(1) man page. -""") - - copr_parser.add_argument( - '--nowait', action='store_true', default=False, - help="Don't wait on build") - copr_parser.add_argument( - 'project', nargs=1, help='Name of the project in format USER/PROJECT') - copr_parser.set_defaults(command=self.copr_build) - - def register_switch_branch(self): - """Register the switch-branch target""" - - switch_branch_parser = self.subparsers.add_parser( - 'switch-branch', help='Work with branches', - description='This command can switch to a local git branch. If ' - 'provided with a remote branch name that does not ' - 'have a local match it will create one. It can also ' - 'be used to list the existing local and remote ' - 'branches.') - switch_branch_parser.add_argument( - 'branch', nargs='?', help='Branch name to switch to') - switch_branch_parser.add_argument( - '-l', '--list', action='store_true', - help='List both remote-tracking branches and local branches') - switch_branch_parser.add_argument( - '--fetch', help='Fetch new data from remote before switch', - action='store_true', dest='fetch') - switch_branch_parser.set_defaults(command=self.switch_branch) - - def register_tag(self): - """Register the tag target""" - - tag_parser = self.subparsers.add_parser( - 'tag', help='Management of git tags', - description='This command uses git to create, list, or delete ' - 'tags.') - tag_parser.add_argument( - '-f', '--force', default=False, - action='store_true', help='Force the creation of the tag') - tag_parser.add_argument( - '-m', '--message', default=None, - help='Use the given as the tag message') - tag_parser.add_argument( - '-c', '--clog', default=False, action='store_true', - help='Generate the tag message from the spec changelog section') - tag_parser.add_argument( - '--raw', action='store_true', default=False, - help='Make the clog raw') - tag_parser.add_argument( - '-F', '--file', default=None, - help='Take the tag message from the given file') - tag_parser.add_argument( - '-l', '--list', default=False, action='store_true', - help='List all tags with a given pattern, or all if not pattern ' - 'is given') - tag_parser.add_argument( - '-d', '--delete', default=False, action='store_true', - help='Delete a tag') - tag_parser.add_argument( - 'tag', nargs='?', default=None, help='Name of the tag') - tag_parser.set_defaults(command=self.tag) - - def register_unused_patches(self): - """Register the unused-patches target""" - - unused_patches_parser = self.subparsers.add_parser( - 'unused-patches', - help='Print list of patches not referenced by name in the ' - 'specfile') - unused_patches_parser.set_defaults(command=self.unused_patches) - - def register_upload(self): - """Register the upload target""" - - upload_parser = self.subparsers.add_parser( - 'upload', parents=[self.new_sources_parser], - conflict_handler='resolve', help='Upload source files', - description='This command will add a new source archive to the ' - 'lookaside cache. The sources and .gitignore file ' - 'will be updated with the new file(s).') - upload_parser.set_defaults(command=self.new_sources, replace=False) - - def register_verify_files(self): - """Register the verify-files target""" - - verify_files_parser = self.subparsers.add_parser( - 'verify-files', parents=[self.rpm_parser_common], - help='Locally verify %%files section', - description="Locally run 'rpmbuild -bl' to verify the spec file's" - " %files sections. This requires a successful run of " - "'{0} install' in advance.".format(self.name)) - verify_files_parser.set_defaults(command=self.verify_files) - - def register_verrel(self): - - verrel_parser = self.subparsers.add_parser( - 'verrel', help='Print the name-version-release') - verrel_parser.set_defaults(command=self.verrel) - - def register_container_build(self): - self.container_build_parser = \ - self.subparsers.add_parser('container-build', - help='build a container') - self.container_build_parser.add_argument('--repo-url', - metavar="URL", - help=("URL of yum repo file"), - nargs='*') - osbs_group = self.container_build_parser.add_argument_group('osbs') - osbs_group.add_argument('--osbs-config', - help="path to file with configuration of osbs", - metavar="PATH", - default=OSBS_DEFAULT_CONF_FILE) - osbs_group.add_argument('--instance', - help=("use specific instance specified " - "by section name in config"), - metavar="SECTION", default="default") - koji_group = self.container_build_parser.add_argument_group('koji') - koji_group.add_argument('--scratch', - help='Scratch build', - action="store_true") - - self.container_build_parser.add_argument( - '--target', - help='Override the default target', - default=None) - self.container_build_parser.add_argument( - '--build-with', - help='Build container with specified builder type. Default is koji', - dest="build_with", - choices=("koji", "osbs"), - default="koji") - self.container_build_parser.add_argument( - '--nowait', - action='store_true', - default=False, - help="Don't wait on build") - - self.container_build_parser.set_defaults(command=self.container_build) - - def register_container_build_setup(self): - self.container_build_setup_parser = \ - self.subparsers.add_parser('container-build-setup', - help='set options for container-build') - group = self.container_build_setup_parser.add_mutually_exclusive_group(required=True) - group.add_argument( - '--get-autorebuild', - help='Get autorebuild value', - action='store_true', - default=None) - group.add_argument( - '--set-autorebuild', - help='Turn autorebuilds on/off', - choices=('true', 'false'), - default=None) - self.container_build_setup_parser.set_defaults( - command=self.container_build_setup) - - # All the command functions go here - def usage(self): - self.parser.print_help() - - def build(self, sets=None): - # We may have gotten arches by way of scratch build, so handle them - arches = None - if hasattr(self.args, 'arches'): - arches = self.args.arches - # Place holder for if we build with an uploaded srpm or not - url = None - # See if this is a chain or not - chain = None - if hasattr(self.args, 'chain'): - chain = self.args.chain - # Need to do something with BUILD_FLAGS or KOJI_FLAGS here for compat - if self.args.target: - self.cmd._target = self.args.target - # handle uploading the srpm if we got one - if hasattr(self.args, 'srpm') and self.args.srpm: - # See if we need to generate the srpm first - if self.args.srpm == 'CONSTRUCT': - self.log.debug('Generating an srpm') - self.srpm() - self.args.srpm = '%s.src.rpm' % self.cmd.nvr - # Figure out if we want a verbose output or not - callback = None - if not self.args.q: - callback = self._progress_callback - # define a unique path for this upload. Stolen from /usr/bin/koji - uniquepath = ('cli-build/%r.%s' - % (time.time(), - ''.join([random.choice(string.ascii_letters) - for i in range(8)]))) - # Should have a try here, not sure what errors we'll get yet though - self.cmd.koji_upload(self.args.srpm, uniquepath, callback=callback) - if not self.args.q: - # print an extra blank line due to callback oddity - print('') - url = '%s/%s' % (uniquepath, os.path.basename(self.args.srpm)) - # nvr_check option isn't set by all commands which calls this - # function so handle it as an optional argument - nvr_check = True - if hasattr(self.args, 'nvr_check'): - nvr_check = self.args.nvr_check - task_id = self.cmd.build(self.args.skip_tag, self.args.scratch, - self.args.background, url, chain, arches, - sets, nvr_check) - - # Log out of the koji session - self.cmd.kojisession.logout() - - if self.args.nowait: - return - - # Pass info off to our koji task watcher - return self._watch_koji_tasks(self.cmd.kojisession, [task_id]) - - def chainbuild(self): - if self.cmd.module_name in self.args.package: - raise Exception('%s must not be in the chain' % self.cmd.module_name) - - # make sure we didn't get an empty chain - if self.args.package == [':']: - raise Exception('Must provide at least one dependency build') - - # Break the chain up into sections - sets = False - urls = [] - build_set = [] - self.log.debug('Processing chain %s' % ' '.join(self.args.package)) - for component in self.args.package: - if component == ':': - # We've hit the end of a set, add the set as a unit to the - # url list and reset the build_set. - urls.append(build_set) - self.log.debug('Created a build set: %s', ' '.join(build_set)) - build_set = [] - sets = True - else: - # Figure out the scm url to build from package name - hash = self.cmd.get_latest_commit(component, self.cmd.branch_merge) - url = self.cmd.anongiturl % {'module': component} + '#%s' % hash - # If there are no ':' in the chain list, treat each object as - # an individual chain - if ':' in self.args.package: - build_set.append(url) - else: - urls.append([url]) - self.log.debug('Created a build set: %s', url) - # Take care of the last build set if we have one - if build_set: - self.log.debug('Created a build set: %s', ' '.join(build_set)) - urls.append(build_set) - # See if we ended in a : making our last build it's own group - if self.args.package[-1] == ':': - self.log.debug('Making the last build its own set.') - urls.append([]) - # pass it off to build - self.args.chain = urls - self.args.skip_tag = False - self.args.scratch = False - return self.build(sets=sets) - - def clean(self): - dry = False - useignore = True - if self.args.dry_run: - dry = True - if self.args.x: - useignore = False - return self.cmd.clean(dry, useignore) - - def clog(self): - self.cmd.clog(raw=self.args.raw) - - def clone(self): - if self.args.branches: - self.cmd.clone_with_dirs(self.args.module[0], - anon=self.args.anonymous, - target=self.args.clone_target) - else: - self.cmd.clone(self.args.module[0], - branch=self.args.branch, - anon=self.args.anonymous, - target=self.args.clone_target) - - def commit(self): - if self.args.clog: - self.cmd.clog(self.args.raw) - self.args.file = os.path.abspath(os.path.join(self.args.path, - 'clog')) - try: - self.cmd.commit(self.args.message, self.args.file, - self.args.files, self.args.signoff) - except Exception: - if self.args.tag: - self.log.error('Could not commit, will not tag!') - if self.args.push: - self.log.error('Could not commit, will not push!') - raise - - try: - if self.args.tag: - tagname = self.cmd.nvr - self.cmd.add_tag(tagname, True, self.args.message, - self.args.file) - except Exception: - if self.args.push: - self.log.error('Could not tag, will not push!') - raise - - if self.args.push: - self.push() - - def compile(self): - arch = None - short = False - nocheck = False - if self.args.arch: - arch = self.args.arch - if self.args.short_circuit: - short = True - if self.args.nocheck: - nocheck = True - self.cmd.compile(arch=arch, short=short, - builddir=self.args.builddir, nocheck=nocheck) - - def container_build(self): - if self.args.build_with == "koji": - self.container_build_koji() - elif self.args.build_with == "osbs": - self.container_build_osbs() - - def container_build_koji(self): - target_override = False - # Override the target if we were supplied one - if self.args.target: - self.cmd._target = self.args.target - target_override = True - - opts = {"scratch": self.args.scratch, - "quiet": self.args.q, - "yum_repourls": self.args.repo_url, - "git_branch": self.cmd.branch_merge} - - section_name = "%s.container-build" % self.name - err_msg = "Missing {option} option in [{plugin.section}] section. "\ - "Using {option} from [{root.section}]" - err_args = {"plugin.section": section_name, "root.section": self.name} - - if self.config.has_option(section_name, "kojiconfig"): - kojiconfig = self.config.get(section_name, "kojiconfig") - else: - err_args["option"] = "kojiconfig" - self.log.debug(err_msg % err_args) - kojiconfig = self.config.get(self.name, "kojiconfig") - - if self.config.has_option(section_name, "build_client"): - build_client = self.config.get(section_name, "build_client") - else: - err_args["option"] = "kojiconfig" - self.log.debug(err_msg % err_args) - build_client = self.config.get(self.name, "build_client") - - self.cmd.container_build_koji(target_override, opts=opts, - kojiconfig=kojiconfig, - build_client=build_client, - koji_task_watcher=self._watch_koji_tasks, - nowait=self.args.nowait) - - def container_build_osbs(self): - target_override = False - # Override the target if we were supplied one - if self.args.target: - self.cmd._target = self.args.target - target_override = True - - self.cmd.osbs_build( - config_file=self.args.osbs_config, - config_section=self.args.instance, - target_override=target_override, - yum_repourls=self.args.repo_url, - nowait=self.args.nowait - ) - - def container_build_setup(self): - self.cmd.container_build_setup(get_autorebuild=self.args.get_autorebuild, - set_autorebuild=self.args.set_autorebuild) - - def copr_build(self): - self.log.debug('Generating an srpm') - self.args.hash = None - self.srpm() - srpm_name = '%s.src.rpm' % self.cmd.nvr - self.cmd.copr_build(self.args.project[0], srpm_name, self.args.nowait) - - def diff(self): - self.cmd.diff(self.args.cached, self.args.files) - - def gimmespec(self): - print(self.cmd.spec) - - def gitbuildhash(self): - print(self.cmd.gitbuildhash(self.args.build)) - - def giturl(self): - print(self.cmd.giturl()) - - def import_srpm(self): - uploadfiles = self.cmd.import_srpm(self.args.srpm) - if uploadfiles: - self.cmd.upload(uploadfiles, replace=True) - if not self.args.skip_diffs: - self.cmd.diff(cached=True) - self.log.info('--------------------------------------------') - self.log.info("New content staged and new sources uploaded.") - self.log.info("Commit if happy or revert with: git reset --hard HEAD") - - def install(self): - self.cmd.install(arch=self.args.arch, - short=self.args.short_circuit, - builddir=self.args.builddir, - nocheck=self.args.nocheck) - - def lint(self): - self.cmd.lint(self.args.info, self.args.rpmlintconf) - - def local(self): - self.cmd.local(arch=self.args.arch, hashtype=self.args.hash, - builddir=self.args.builddir) - - def mockbuild(self): - try: - self.cmd.sources() - except Exception as e: - self.log.error('Could not download sources: %s' % e) - sys.exit(1) - - mockargs = [] - - if self.args.no_clean or self.args.no_clean_all: - mockargs.append('--no-clean') - - if self.args.no_cleanup_after or self.args.no_clean_all: - mockargs.append('--no-cleanup-after') - - # Pick up any mockargs from the env - try: - mockargs += os.environ['MOCKARGS'].split() - except KeyError: - # there were no args - pass - try: - self.cmd.mockbuild(mockargs, self.args.root, - hashtype=self.args.hash) - except Exception as e: - self.log.error('Could not run mockbuild: %s' % e) - sys.exit(1) - - def mock_config(self): - try: - print(self.cmd.mock_config(self.args.target, self.args.arch)) - except Exception as e: - self.log.error('Could not generate the mock config: %s' % e) - sys.exit(1) - - def new(self): - print(self.cmd.new()) - - def new_sources(self): - # Check to see if the files passed exist - for file in self.args.files: - if not os.path.isfile(file): - raise Exception('Path does not exist or is ' - 'not a file: %s' % file) - self.cmd.upload(self.args.files, replace=self.args.replace) - self.log.info("Source upload succeeded. Don't forget to commit the " - "sources file") - - def patch(self): - self.cmd.patch(self.args.suffix, rediff=self.args.rediff) - - def prep(self): - self.cmd.prep(arch=self.args.arch, builddir=self.args.builddir) - - def pull(self): - self.cmd.pull(rebase=self.args.rebase, - norebase=self.args.no_rebase) - - def push(self): - self.cmd.push(getattr(self.args, 'force', False)) - - def scratch_build(self): - # A scratch build is just a build with --scratch - self.args.scratch = True - self.args.skip_tag = False - return self.build() - - def sources(self): - self.cmd.sources(self.args.outdir) - - def srpm(self): - self.cmd.sources() - self.cmd.srpm(hashtype=self.args.hash) - - def switch_branch(self): - if self.args.branch: - self.cmd.switch_branch(self.args.branch, self.args.fetch) - else: - (locals, remotes) = self.cmd._list_branches(self.args.fetch) - # This is some ugly stuff here, but trying to emulate - # the way git branch looks - locals = [' %s ' % branch for branch in locals] - local_branch = self.cmd.repo.active_branch.name - locals[locals.index(' %s ' % - local_branch)] = '* %s' % local_branch - print('Locals:\n%s\nRemotes:\n %s' % - ('\n'.join(locals), '\n '.join(remotes))) - - def tag(self): - if self.args.list: - self.cmd.list_tag(self.args.tag) - elif self.args.delete: - self.cmd.delete_tag(self.args.tag) - else: - filename = self.args.file - tagname = self.args.tag - if not tagname or self.args.clog: - if not tagname: - tagname = self.cmd.nvr - if self.args.clog: - self.cmd.clog(self.args.raw) - filename = 'clog' - self.cmd.add_tag(tagname, self.args.force, - self.args.message, filename) - - def unused_patches(self): - unused = self.cmd.unused_patches() - print('\n'.join(unused)) - - def verify_files(self): - self.cmd.verify_files(builddir=self.args.builddir) - - def verrel(self): - print('%s-%s-%s' % (self.cmd.module_name, self.cmd.ver, - self.cmd.rel)) - - # Other class stuff goes here - # The next 6 functions come from the koji project, from /usr/bin/koji - # They should be in a library somewhere, but I have to steal them. - # The code is licensed LGPLv2.1 and thus my (slightly) derived code - # is as well. - def _display_tasklist_status(self, tasks): - free = 0 - open = 0 - failed = 0 - done = 0 - for task_id in tasks.keys(): - status = tasks[task_id].info['state'] - if status == koji.TASK_STATES['FAILED']: - failed += 1 - elif status in (koji.TASK_STATES['CLOSED'], - koji.TASK_STATES['CANCELED']): - done += 1 - elif status in (koji.TASK_STATES['OPEN'], - koji.TASK_STATES['ASSIGNED']): - open += 1 - elif status == koji.TASK_STATES['FREE']: - free += 1 - self.log.info(" %d free %d open %d done %d failed" % - (free, open, done, failed)) - - def _display_task_results(self, tasks): - for task in [task for task in tasks.values() if task.level == 0]: - state = task.info['state'] - task_label = task.str() - - if state == koji.TASK_STATES['CLOSED']: - self.log.info('%s completed successfully' % task_label) - elif state == koji.TASK_STATES['FAILED']: - self.log.info('%s failed' % task_label) - elif state == koji.TASK_STATES['CANCELED']: - self.log.info('%s was canceled' % task_label) - else: - # shouldn't happen - self.log.info('%s has not completed' % task_label) - - def _watch_koji_tasks(self, session, tasklist): - if not tasklist: - return - self.log.info('Watching tasks (this may be safely interrupted)...') - # Place holder for return value - rv = 0 - try: - tasks = {} - for task_id in tasklist: - tasks[task_id] = TaskWatcher(task_id, session, self.log, - quiet=self.args.q) - while True: - all_done = True - for task_id, task in tasks.items(): - changed = task.update() - if not task.is_done(): - all_done = False - else: - if changed: - # task is done and state just changed - if not self.args.q: - self._display_tasklist_status(tasks) - if not task.is_success(): - rv = 1 - for child in session.getTaskChildren(task_id): - child_id = child['id'] - if child_id not in tasks.keys(): - tasks[child_id] = TaskWatcher(child_id, - session, - self.log, - task.level + 1, - quiet=self.args.q) - tasks[child_id].update() - # If we found new children, go through the list - # again, in case they have children also - all_done = False - if all_done: - if not self.args.q: - print("") - self._display_task_results(tasks) - break - - time.sleep(1) - except (KeyboardInterrupt): - if tasks: - self.log.info(""" -Tasks still running. You can continue to watch with the '%s watch-task' command. - Running Tasks: - %s""" - % (self.config.get(self.name, 'build_client'), - '\n'.join(['%s: %s' % (t.str(), - t.display_state(t.info)) - for t in tasks.values() - if not t.is_done()]))) - # A ^c should return non-zero so that it doesn't continue - # on to any && commands. - rv = 1 - return rv - - # Stole these three functions from /usr/bin/koji - def _format_size(self, size): - if (size / 1073741824 >= 1): - return "%0.2f GiB" % (size / 1073741824.0) - if (size / 1048576 >= 1): - return "%0.2f MiB" % (size / 1048576.0) - if (size / 1024 >= 1): - return "%0.2f KiB" % (size / 1024.0) - return "%0.2f B" % (size) - - def _format_secs(self, t): - h = t / 3600 - t = t % 3600 - m = t / 60 - s = t % 60 - return "%02d:%02d:%02d" % (h, m, s) - - def _progress_callback(self, uploaded, total, piece, time, total_time): - percent_done = float(uploaded)/float(total) - percent_done_str = "%02d%%" % (percent_done * 100) - data_done = self._format_size(uploaded) - elapsed = self._format_secs(total_time) - - speed = "- B/sec" - if (time): - if (uploaded != total): - speed = self._format_size(float(piece)/float(time)) + "/sec" - else: - speed = self._format_size(float(total)/float(total_time)) + \ - "/sec" - - # write formatted string and flush - sys.stdout.write("[% -36s] % 4s % 8s % 10s % 14s\r" % - ('='*(int(percent_done*36)), - percent_done_str, elapsed, data_done, speed)) - sys.stdout.flush() - - def setupLogging(self, log): - """Setup the various logging stuff.""" - - # Assign the log object to self - self.log = log - - # Add a log filter class - class StdoutFilter(logging.Filter): - - def filter(self, record): - # If the record level is 20 (INFO) or lower, let it through - return record.levelno <= logging.INFO - - # have to create a filter for the stdout stream to filter out WARN+ - myfilt = StdoutFilter() - # Simple format - formatter = logging.Formatter('%(message)s') - stdouthandler = logging.StreamHandler(sys.stdout) - stdouthandler.addFilter(myfilt) - stdouthandler.setFormatter(formatter) - stderrhandler = logging.StreamHandler() - stderrhandler.setLevel(logging.WARNING) - stderrhandler.setFormatter(formatter) - self.log.addHandler(stdouthandler) - self.log.addHandler(stderrhandler) - - def parse_cmdline(self, manpage=False): - """Parse the commandline, optionally make a manpage - - This also sets up self.user - """ - - if manpage: - # Generate the man page - man_name = self.name - if man_name.endswith('.py'): - man_name = man_name[:-3] - man_page = __import__('%s' % man_name) - man_page.generate(self.parser, self.subparsers) - sys.exit(0) - # no return possible - - # Parse the args - self.args = self.parser.parse_args() - if self.args.user: - self.user = self.args.user - else: - self.user = pwd.getpwuid(os.getuid())[0] - - -# Add a class stolen from /usr/bin/koji to watch tasks -# this was cut/pasted from koji, and then modified for local use. -# The formatting is koji style, not the stile of this file. Do not use these -# functions as a style guide. -# This is fragile and hopefully will be replaced by a real kojiclient lib. - - -class TaskWatcher(object): - - def __init__(self, task_id, session, log, level=0, quiet=False): - self.id = task_id - self.session = session - self.info = None - self.level = level - self.quiet = quiet - self.log = log - - # XXX - a bunch of this stuff needs to adapt to different tasks - - def str(self): - if self.info: - label = koji.taskLabel(self.info) - return "%s%d %s" % (' ' * self.level, self.id, label) - else: - return "%s%d" % (' ' * self.level, self.id) - - def __str__(self): - return self.str() - - def get_failure(self): - """Print information about task completion""" - if self.info['state'] != koji.TASK_STATES['FAILED']: - return '' - error = None - try: - self.session.getTaskResult(self.id) - except (xmlrpc_client.Fault, koji.GenericError) as e: - error = e - if error is None: - # print "%s: complete" % self.str() - # We already reported this task as complete in update() - return '' - else: - return '%s: %s' % (error.__class__.__name__, str(error).strip()) - - def update(self): - """Update info and log if needed. Returns True on state change.""" - if self.is_done(): - # Already done, nothing else to report - return False - last = self.info - try: - self.info = self.session.getTaskInfo(self.id, request=True) - except koji.GenericError: - raise Exception("No such task id: %i" % self.id) - state = self.info['state'] - if last: - # compare and note status changes - laststate = last['state'] - if laststate != state: - self.log.info("%s: %s -> %s", - self.str(), self.display_state(last), self.display_state(self.info)) - return True - return False - else: - # First time we're seeing this task, so just show the current state - self.log.info("%s: %s", self.str(), self.display_state(self.info)) - return False - - def is_done(self): - if self.info is None: - return False - state = koji.TASK_STATES[self.info['state']] - return (state in ['CLOSED', 'CANCELED', 'FAILED']) - - def is_success(self): - if self.info is None: - return False - state = koji.TASK_STATES[self.info['state']] - return (state == 'CLOSED') - - def display_state(self, info): - # We can sometimes be passed a task that is not yet open, but - # not finished either. info would be none. - if not info: - return 'unknown' - if info['state'] == koji.TASK_STATES['OPEN']: - if info['host_id']: - host = self.session.getHost(info['host_id']) - return 'open (%s)' % host['name'] - else: - return 'open' - elif info['state'] == koji.TASK_STATES['FAILED']: - return 'FAILED: %s' % self.get_failure() - else: - return koji.TASK_STATES[info['state']].lower() - - -if __name__ == '__main__': - client = cliClient() - client.do_imports() - client.parse_cmdline() - - if not client.args.path: - try: - client.args.path = os.getcwd() - except: - print('Could not get current path, have you deleted it?') - sys.exit(1) - - # setup the logger -- This logger will take things of INFO or DEBUG and - # log it to stdout. Anything above that (WARN, ERROR, CRITICAL) will go - # to stderr. Normal operation will show anything INFO and above. - # Quiet hides INFO, while Verbose exposes DEBUG. In all cases WARN or - # higher are exposed (via stderr). - log = client.site.log - client.setupLogging(log) - - if client.args.v: - log.setLevel(logging.DEBUG) - elif client.args.q: - log.setLevel(logging.WARNING) - else: - log.setLevel(logging.INFO) - - # Run the necessary command - try: - client.args.command() - except KeyboardInterrupt: - pass diff --git a/src/pyrpkg/errors.py b/src/pyrpkg/errors.py deleted file mode 100644 index 5ea5f08..0000000 --- a/src/pyrpkg/errors.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2015 - Red Hat Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - - -"""Custom error classes""" - - -class rpkgError(Exception): - """Our base error class""" - faultCode = 1000 - - -class rpkgAuthError(rpkgError): - """Raised in case of authentication errors""" - faultCode = 1002 - - -class UnknownTargetError(Exception): - faultCode = 1004 - - -class HashtypeMixingError(rpkgError): - """Raised when we try to mix hash types in a sources file""" - def __init__(self, existing_hashtype, new_hashtype): - super(HashtypeMixingError, self).__init__() - - self.existing_hashtype = existing_hashtype - self.new_hashtype = new_hashtype - - -class MalformedLineError(rpkgError): - """Raised when parsing a sources file with malformed lines""" - pass - - -class InvalidHashType(rpkgError): - """Raised when we don't know the requested hash algorithm""" - pass - - -class DownloadError(rpkgError): - """Raised when something went wrong during a download""" - pass - - -class UploadError(rpkgError): - """Raised when something went wrong during an upload""" - pass diff --git a/src/pyrpkg/gitignore.py b/src/pyrpkg/gitignore.py deleted file mode 100644 index db3f3a5..0000000 --- a/src/pyrpkg/gitignore.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) 2015 - Red Hat Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - - -"""Manage a .gitignore file""" - - -import fnmatch -import os - - -class GitIgnore(object): - """A class to manage a .gitignore file""" - def __init__(self, path): - """Constructor - - Args: - path (str): The full path to the .gitignore file. If it does not - exist, the file will be created when running GitIgnore.write() - for the first time. - """ - self.path = path - - # Lines of the .gitignore file, used to check if entries need to be - # added or already exist. - self.__lines = [] - - if os.path.exists(self.path): - with open(self.path, 'r') as f: - for line in f: - self.__lines.append(self.__ensure_newline(line)) - - # Set to True if we end up making any modifications, used to - # prevent unnecessary writes. - self.modified = False - - def __ensure_newline(self, line): - return line if line.endswith('\n') else '%s\n' % line - - def add(self, line): - """Add a line - - Args: - line (str): The line to add to the file. It will not be added if - it already matches an existing line. - """ - if self.match(line): - return - - line = self.__ensure_newline(line) - self.__lines.append(line) - self.modified = True - - def match(self, line): - """Check whether the line matches an existing one - - This uses fnmatch to match against wildcards. - - Args: - line (str): The new line to match against existing ones. - - Returns: - True if the new line matches, False otherwise. - """ - line = line.lstrip('/').rstrip('\n') - - for entry in self.__lines: - entry = entry.lstrip('/').rstrip('\n') - if fnmatch.fnmatch(line, entry): - return True - - return False - - def write(self): - """Write the file to the disk - - This will only actually write if necessary, that is if lines have been - added since the last time the file was written. - """ - if self.modified: - with open(self.path, 'w') as f: - for line in self.__lines: - f.write(line) - - self.modified = False diff --git a/src/pyrpkg/lookaside.py b/src/pyrpkg/lookaside.py deleted file mode 100644 index e257208..0000000 --- a/src/pyrpkg/lookaside.py +++ /dev/null @@ -1,307 +0,0 @@ -# Copyright (c) 2015 - Red Hat Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - - -"""Interact with a lookaside cache - -This module contains everything needed to upload and download source files the -way it is done by Fedora, RHEL, and other distributions maintainers. -""" - - -import hashlib -import io -import logging -import os -import sys - -import pycurl - -from .errors import DownloadError, InvalidHashType, UploadError - - -class CGILookasideCache(object): - """A class to interact with a CGI-based lookaside cache""" - def __init__(self, hashtype, download_url, upload_url, - client_cert=None, ca_cert=None): - """Constructor - - Args: - hashtype (str): The hash algorithm to use for uploads. (e.g 'md5') - download_url (str): The URL used to download source files. - upload_url (str): The URL of the CGI script called when uploading - source files. - client_cert (str, optional): The full path to the client-side - certificate to use for HTTPS authentication. It defaults to - None, in which case no client-side certificate is used. - ca_cert (str, optional): The full path to the CA certificate to - use for HTTPS connexions. (e.g if the server certificate is - self-signed. It defaults to None, in which case the system CA - bundle is used. - """ - self.hashtype = hashtype - self.download_url = download_url - self.upload_url = upload_url - self.client_cert = client_cert - self.ca_cert = ca_cert - - self.log = logging.getLogger(__name__) - - self.download_path = '%(name)s/%(filename)s/%(hash)s/%(filename)s' - - def print_progress(self, to_download, downloaded, to_upload, uploaded): - if not sys.stdout.isatty(): - # Don't print progress if not outputting into TTY. The progress - # output is not useful in logs. - return - - if to_download > 0: - done = downloaded / to_download - - elif to_upload > 0: - done = uploaded / to_upload - - else: - return - - done_chars = int(done * 72) - remain_chars = 72 - done_chars - done = int(done * 1000) / 10.0 - - p = "\r%s%s %s%%" % ("#" * done_chars, " " * remain_chars, done) - sys.stdout.write(p) - sys.stdout.flush() - - def hash_file(self, filename, hashtype=None): - """Compute the hash of a file - - Args: - filename (str): The full path to the file. It is assumed to exist. - hashtype (str, optional): The hash algorithm to use. (e.g 'md5') - This defaults to the hashtype passed to the constructor. - - Returns: - The hash digest. - """ - if hashtype is None: - hashtype = self.hashtype - - try: - sum = hashlib.new(hashtype) - - except ValueError: - raise InvalidHashType(hashtype) - - with open(filename, 'rb') as f: - chunk = f.read(8192) - - while chunk: - sum.update(chunk) - chunk = f.read(8192) - - return sum.hexdigest() - - def file_is_valid(self, filename, hash, hashtype=None): - """Ensure the file is correct - - Args: - filename (str): The full path to the file. It is assumed to exist. - hash (str): The known good hash of the file. - hashtype (str, optional): The hash algorithm to use. (e.g 'md5') - This defaults to the hashtype passed to the constructor. - - Returns: - True if the file is valid, False otherwise. - """ - sum = self.hash_file(filename, hashtype) - return sum == hash - - def download(self, name, filename, hash, outfile, hashtype=None, **kwargs): - """Download a source file - - Args: - name (str): The name of the module. (usually the name of the SRPM) - filename (str): The name of the file to download. - hash (str): The known good hash of the file. - outfile (str): The full path where to save the downloaded file. - hashtype (str, optional): The hash algorithm. (e.g 'md5') - This defaults to the hashtype passed to the constructor. - **kwargs: Additional keyword arguments. They will be used when - contructing the full URL to the file to download. - """ - if hashtype is None: - hashtype = self.hashtype - - if os.path.exists(outfile): - if self.file_is_valid(outfile, hash, hashtype=hashtype): - return - - self.log.info("Downloading %s", filename) - urled_file = filename.replace(' ', '%20') - - path_dict = {'name': name, 'filename': urled_file, 'hash': hash, - 'hashtype': hashtype} - path_dict.update(kwargs) - path = self.download_path % path_dict - url = '%s/%s' % (self.download_url, path) - self.log.debug("Full url: %s" % url) - - with open(outfile, 'wb') as f: - c = pycurl.Curl() - c.setopt(pycurl.URL, url) - c.setopt(pycurl.HTTPHEADER, ['Pragma:']) - c.setopt(pycurl.NOPROGRESS, False) - c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress) - c.setopt(pycurl.OPT_FILETIME, True) - c.setopt(pycurl.WRITEDATA, f) - - try: - c.perform() - tstamp = c.getinfo(pycurl.INFO_FILETIME) - status = c.getinfo(pycurl.RESPONSE_CODE) - - except Exception as e: - raise DownloadError(e) - - finally: - c.close() - - # Get back a new line, after displaying the download progress - sys.stdout.write('\n') - sys.stdout.flush() - - if status != 200: - self.log.info('Remove downloaded invalid file %s', outfile) - os.remove(outfile) - raise DownloadError('Server returned status code %d' % status) - - os.utime(outfile, (tstamp, tstamp)) - - if not self.file_is_valid(outfile, hash, hashtype=hashtype): - raise DownloadError('%s failed checksum' % filename) - - def remote_file_exists(self, name, filename, hash): - """Verify whether a file exists on the lookaside cache - - Args: - name: The name of the module. (usually the name of the SRPM) - filename: The name of the file to check for. - hash: The known good hash of the file. - """ - post_data = [('name', name), - ('%ssum' % self.hashtype, hash), - ('filename', filename)] - - with io.BytesIO() as buf: - c = pycurl.Curl() - c.setopt(pycurl.URL, self.upload_url) - c.setopt(pycurl.WRITEFUNCTION, buf.write) - c.setopt(pycurl.HTTPPOST, post_data) - - if self.client_cert is not None: - if os.path.exists(self.client_cert): - c.setopt(pycurl.SSLCERT, self.client_cert) - else: - self.log.warning("Missing certificate: %s" - % self.client_cert) - - if self.ca_cert is not None: - if os.path.exists(self.ca_cert): - c.setopt(pycurl.CAINFO, self.ca_cert) - else: - self.log.warning("Missing certificate: %s" % self.ca_cert) - - try: - c.perform() - status = c.getinfo(pycurl.RESPONSE_CODE) - - except Exception as e: - raise UploadError(e) - - finally: - c.close() - - output = buf.getvalue().strip() - - if status != 200: - raise UploadError(output) - - # Lookaside CGI script returns these strings depending on whether - # or not the file exists: - if output == b'Available': - return True - - if output == b'Missing': - return False - - # Something unexpected happened - self.log.debug(output) - raise UploadError('Error checking for %s at %s' - % (filename, self.upload_url)) - - def upload(self, name, filepath, hash): - """Upload a source file - - Args: - name (str): The name of the module. (usually the name of the SRPM) - filepath (str): The full path to the file to upload. - hash (str): The known good hash of the file. - """ - filename = os.path.basename(filepath) - - if self.remote_file_exists(name, filename, hash): - self.log.info("File already uploaded: %s" % filepath) - return - - self.log.info("Uploading: %s" % filepath) - post_data = [('name', name), - ('%ssum' % self.hashtype, hash), - ('file', (pycurl.FORM_FILE, filepath))] - - with io.BytesIO() as buf: - c = pycurl.Curl() - c.setopt(pycurl.URL, self.upload_url) - c.setopt(pycurl.NOPROGRESS, False) - c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress) - c.setopt(pycurl.WRITEFUNCTION, buf.write) - c.setopt(pycurl.HTTPPOST, post_data) - - if self.client_cert is not None: - if os.path.exists(self.client_cert): - c.setopt(pycurl.SSLCERT, self.client_cert) - else: - self.log.warning("Missing certificate: %s" - % self.client_cert) - - if self.ca_cert is not None: - if os.path.exists(self.ca_cert): - c.setopt(pycurl.CAINFO, self.ca_cert) - else: - self.log.warning("Missing certificate: %s" % self.ca_cert) - - try: - c.perform() - status = c.getinfo(pycurl.RESPONSE_CODE) - - except Exception as e: - raise UploadError(e) - - finally: - c.close() - - output = buf.getvalue().strip() - - # Get back a new line, after displaying the download progress - sys.stdout.write('\n') - sys.stdout.flush() - - if status != 200: - raise UploadError(output) - - if output: - self.log.debug(output) diff --git a/src/pyrpkg/sources.py b/src/pyrpkg/sources.py deleted file mode 100644 index a443f38..0000000 --- a/src/pyrpkg/sources.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -Our so-called sources file is simple text-based line-oriented file format. - -Each line represents one source file and is in the same format as the output -of commands like `md5sum --tag filename`: - - hashtype (filename) = hash - -To preserve backwards compatibility, lines can also be in the older format, -which corresponds to the output of commands like `md5sum filename`: - - hash filename - -This module implements a simple API to read these files, parse lines into -entries, and write these entries to the file in the proper format. -""" - - -import os -import re - -from .errors import HashtypeMixingError, MalformedLineError - - -LINE_PATTERN = re.compile( - r'^(?P[^ ]+?) \((?P[^ )]+?)\) = (?P[^ ]+?)$') - - -class SourcesFile(object): - def __init__(self, sourcesfile, entry_type, replace=False): - self.sourcesfile = sourcesfile - self.entry_type = {'old': SourceFileEntry, - 'bsd': BSDSourceFileEntry}[entry_type] - self.entries = [] - - if not replace: - if not os.path.exists(sourcesfile): - return - - with open(sourcesfile) as f: - for line in f: - entry = self.parse_line(line) - - if entry and entry not in self.entries: - self.entries.append(entry) - - def __contains__(self, filename): - for entry in self.entries: - if entry.file == filename: - return True - return False - - def parse_line(self, line): - stripped = line.strip() - - if not stripped: - return - - m = LINE_PATTERN.match(stripped) - if m is not None: - return self.entry_type(m.group('hashtype'), m.group('file'), - m.group('hash')) - - # Try falling back on the old format - try: - hash, file = stripped.split(' ', 1) - - except ValueError: - raise MalformedLineError(line) - - return self.entry_type('md5', file, hash) - - def add_entry(self, hashtype, file, hash): - entry = self.entry_type(hashtype, file, hash) - - for e in self.entries: - if entry.hashtype != e.hashtype: - raise HashtypeMixingError(e.hashtype, entry.hashtype) - - if entry == e: - return - - self.entries.append(entry) - - def write(self): - with open(self.sourcesfile, 'w') as f: - for entry in self.entries: - f.write(str(entry)) - - -class SourceFileEntry(object): - def __init__(self, hashtype, file, hash): - self.hashtype = hashtype.lower() - self.hash = hash - self.file = file - - def __str__(self): - return '%s %s\n' % (self.hash, self.file) - - def __eq__(self, other): - return ((self.hashtype, self.hash, self.file) == - (other.hashtype, other.hash, other.file)) - - -class BSDSourceFileEntry(SourceFileEntry): - def __str__(self): - return '%s (%s) = %s\n' % (self.hashtype.upper(), self.file, - self.hash) diff --git a/src/pyrpkg/utils.py b/src/pyrpkg/utils.py deleted file mode 100644 index 03606a4..0000000 --- a/src/pyrpkg/utils.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2015 - Red Hat Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - - -"""Miscellaneous utilities - -This module contains a bunch of utilities used elsewhere in pyrpkg. -""" - - -import warnings - -import os -import six - -if six.PY3: - def u(s): - return s - - getcwd = os.getcwd -else: - def u(s): - return s.decode('utf-8') - - getcwd = os.getcwdu - -warnings.simplefilter('always', DeprecationWarning) - - -class cached_property(property): - """A property caching its return value - - This is pretty much the same as a normal Python property, except that the - decorated function is called only once. Its return value is then saved, - subsequent calls will return it without executing the function any more. - - Example: - >>> class Foo(object): - ... @cached_property - ... def bar(self): - ... print("Executing Foo.bar...") - ... return 42 - ... - >>> f = Foo() - >>> f.bar - Executing Foo.bar... - 42 - >>> f.bar - 42 - """ - def __get__(self, inst, type=None): - try: - return getattr(inst, '_%s' % self.fget.__name__) - except AttributeError: - v = super(cached_property, self).__get__(inst, type) - setattr(inst, '_%s' % self.fget.__name__, v) - return v - - -def warn_deprecated(clsname, oldname, newname): - """Emit a deprecation warning - - Args: - clsname (str): The name of the class which has its attribute - deprecated. - oldname (str): The name of the deprecated attribute. - newname (str): The name of the new attribute, which should be used - instead. - """ - warnings.warn( - "%s.%s is deprecated and will be removed eventually.\n Please " - "use %s.%s instead." % (clsname, oldname, clsname, newname), - DeprecationWarning, stacklevel=3) - - -def _log_value(log_func, value, level, indent, suffix=''): - offset = ' ' * level * indent - log_func(''.join([offset, str(value), suffix])) - - -def log_result(log_func, result, level=0, indent=2): - if isinstance(result, list): - for item in result: - log_result(log_func, item, level) - elif isinstance(result, dict): - for key, value in result.items(): - _log_value(log_func, key, level, indent, ':') - log_result(log_func, value, level+1) - else: - _log_value(log_func, result, level, indent) diff --git a/src/rpkg b/src/rpkg deleted file mode 100755 index 6cd10e2..0000000 --- a/src/rpkg +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/python -# rpkg - a script to interact with the Red Hat Packaging system -# -# Copyright (C) 2011 Red Hat Inc. -# Author(s): Jesse Keating -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - -import pyrpkg -import pyrpkg.cli -import pyrpkg.utils -import os -import sys -import logging -from six.moves import configparser -import argparse - -# Setup an argparser and parse the known commands to get the config file -parser = argparse.ArgumentParser(add_help=False) -parser.add_argument('-C', '--config', help='Specify a config file to use', - default='/etc/rpkg/rpkg.conf') - -(args, other) = parser.parse_known_args() - -# Make sure we have a sane config file -if not os.path.exists(args.config) and not other[-1] in ['--help', '-h']: - sys.stderr.write('Invalid config file %s\n' % args.config) - sys.exit(1) - -# Setup a configuration object and read config file data -config = configparser.SafeConfigParser() -config.read(args.config) - -client = pyrpkg.cli.cliClient(config) -client.do_imports() -client.parse_cmdline() - -if not client.args.path: - try: - client.args.path = pyrpkg.utils.getcwd() - except: - print('Could not get current path, have you deleted it?') - sys.exit(1) - -# setup the logger -- This logger will take things of INFO or DEBUG and -# log it to stdout. Anything above that (WARN, ERROR, CRITICAL) will go -# to stderr. Normal operation will show anything INFO and above. -# Quiet hides INFO, while Verbose exposes DEBUG. In all cases WARN or -# higher are exposed (via stderr). -log = pyrpkg.log -client.setupLogging(log) - -if client.args.v: - log.setLevel(logging.DEBUG) -elif client.args.q: - log.setLevel(logging.WARNING) -else: - log.setLevel(logging.INFO) - -# Run the necessary command -try: - sys.exit(client.args.command()) -except KeyboardInterrupt: - pass diff --git a/src/rpkg.bash b/src/rpkg.bash deleted file mode 100644 index 3dce4f7..0000000 --- a/src/rpkg.bash +++ /dev/null @@ -1,321 +0,0 @@ -# rpkg bash completion - -_rpkg() -{ - COMPREPLY=() - - in_array() - { - local i - for i in $2; do - [[ $i = $1 ]] && return 0 - done - return 1 - } - - _filedir_exclude_paths() - { - _filedir "$@" - for ((i=0; i<=${#COMPREPLY[@]}; i++)); do - [[ ${COMPREPLY[$i]} =~ /?\.git/? ]] && unset COMPREPLY[$i] - done - } - - local cur prev - # _get_comp_words_by_ref is in bash-completion >= 1.2, which EL-5 lacks. - if type _get_comp_words_by_ref &>/dev/null; then - _get_comp_words_by_ref cur prev - else - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - fi - - # global options - - local options="--help -v -q" - local options_value="--dist --user --path" - local commands="build chain-build ci clean clog clone co container-build container-build-config commit compile copr-build diff gimmespec giturl help \ - gitbuildhash import install lint local mockbuild mock-config new new-sources patch prep pull push scratch-build sources \ - srpm switch-branch tag unused-patches upload verify-files verrel" - - # parse main options and get command - - local command= - local command_first= - local path= - - local i w - for (( i = 0; i < ${#COMP_WORDS[*]} - 1; i++ )); do - w="${COMP_WORDS[$i]}" - # option - if [[ ${w:0:1} = - ]]; then - if in_array "$w" "$options_value"; then - ((i++)) - [[ "$w" = --path ]] && path="${COMP_WORDS[$i]}" - fi - # command - elif in_array "$w" "$commands"; then - command="$w" - command_first=$((i+1)) - break - fi - done - - # complete base options - - if [[ -z $command ]]; then - if [[ $cur == -* ]]; then - COMPREPLY=( $(compgen -W "$options $options_value" -- "$cur") ) - return 0 - fi - - case "$prev" in - --config) - _filedir_exclude_paths - ;; - --dist) - ;; - --user|-u) - ;; - --path) - _filedir_exclude_paths - ;; - *) - COMPREPLY=( $(compgen -W "$commands" -- "$cur") ) - ;; - esac - - return 0 - fi - - # parse command specific options - - local options= - local options_target= options_arches= options_branch= options_string= options_file= options_dir= options_srpm= - local after= after_more= - - case $command in - help|gimmespec|gitbuildhash|giturl|lint|new|unused-patches|verrel) - ;; - build) - options="--nowait --background --skip-tag --scratch --md5" - options_arches="--arches" - options_srpm="--srpm" - options_target="--target" - ;; - chain-build) - options="--nowait --background" - options_target="--target" - after="package" - after_more=true - ;; - clean) - options="--dry-run -x" - ;; - clog) - options="--raw" - ;; - clone|co) - options="--branches --anonymous" - options_branch="-b" - after="package" - ;; - container-build) - options="--scratch --nowait" - options_target="--target" - options_string="--repo-url" - ;; - container-build-config) - options="--get-autorebuild" - options_bool="--set-autorebuild" - ;; - commit|ci) - options="--push --clog --raw --tag" - options_string="--message" - options_file="--file" - after="file" - after_more=true - ;; - compile|install) - options="--short-circuit --nocheck" - options_arch="--arch" - options_dir="--builddir" - ;; - copr-build) - options="--nowait" - after="package" - after_more=true - ;; - diff) - options="--cached" - after="file" - after_more=true - ;; - import) - options="--create" - options_branch="--branch" - after="srpm" - ;; - lint) - options="--info" - options_file="--rpmlintconf" - ;; - local) - options="--md5" - options_arch="--arch" - options_dir="--builddir" - ;; - mock-config) - options="--target" - options_arch="--arch" - ;; - mockbuild) - options="--md5 --no-clean --no-cleanup-after --no-clean-all" - options_mroot="--root" - ;; - patch) - options="--rediff" - options_string="--suffix" - ;; - prep|verify-files) - options_arch="--arch" - options_dir="--builddir" - ;; - pull) - options="--rebase --no-rebase" - ;; - push) - options="--force" - ;; - scratch-build) - options="--nowait --background --md5" - options_target="--target" - options_arches="--arches" - options_srpm="--srpm" - ;; - sources) - options_dir="--outdir" - ;; - srpm) - options="--md5" - ;; - switch-branch) - options="--list" - after="branch" - ;; - tag) - options="--clog --raw --force --list --delete" - options_string="--message" - options_file="--file" - after_more=true - ;; - upload|new-sources) - after="file" - after_more=true - ;; - esac - - local all_options="--help $options" - local all_options_value="$options_target $options_arches $options_branch $options_string $options_file $options_dir $options_srpm $options_bool" - - # count non-option parameters - - local i w - local last_option= - local after_counter=0 - for (( i = $command_first; i < ${#COMP_WORDS[*]} - 1; i++)); do - w="${COMP_WORDS[$i]}" - if [[ ${w:0:1} = - ]]; then - if in_array "$w" "$all_options"; then - last_option="$w" - continue - elif in_array "$w" "$all_options_value"; then - last_option="$w" - ((i++)) - continue - fi - fi - in_array "$last_option" "$options_arches" || ((after_counter++)) - done - - # completion - - if [[ -n $options_target ]] && in_array "$prev" "$options_target"; then - COMPREPLY=( $(compgen -W "$(_rpkg_target)" -- "$cur") ) - - elif [[ -n $options_arches ]] && in_array "$last_option" "$options_arches"; then - COMPREPLY=( $(compgen -W "$(_rpkg_arch) $all_options" -- "$cur") ) - - elif [[ -n $options_srpm ]] && in_array "$prev" "$options_srpm"; then - _filedir_exclude_paths "*.src.rpm" - - elif [[ -n $options_branch ]] && in_array "$prev" "$options_branch"; then - COMPREPLY=( $(compgen -W "$(_rpkg_branch "$path")" -- "$cur") ) - - elif [[ -n $options_file ]] && in_array "$prev" "$options_file"; then - _filedir_exclude_paths - - elif [[ -n $options_dir ]] && in_array "$prev" "$options_dir"; then - _filedir_exclude_paths -d - - elif [[ -n $options_bool ]] && in_array "$prev" "$options_bool"; then - COMPREPLY=( $(compgen -W "true false" -- "$cur") ) - - elif [[ -n $options_string ]] && in_array "$prev" "$options_string"; then - COMPREPLY=( ) - - else - local after_options= - - if [[ $after_counter -eq 0 ]] || [[ $after_more = true ]]; then - case $after in - file) _filedir_exclude_paths ;; - srpm) _filedir_exclude_paths "*.src.rpm" ;; - branch) after_options="$(_rpkg_branch "$path")" ;; - package) after_options="$(_rpkg_package "$cur")";; - esac - fi - - if [[ $cur != -* ]]; then - all_options= - all_options_value= - fi - - COMPREPLY+=( $(compgen -W "$all_options $all_options_value $after_options" -- "$cur" ) ) - fi - - return 0 -} && -complete -F _rpkg rpkg - -_rpkg_target() -{ - koji list-targets --quiet 2>/dev/null | cut -d" " -f1 -} - -_rpkg_arch() -{ - echo "i386 x86_64 ppc ppc64 s390 s390x sparc sparc64" -} - -_rpkg_branch() -{ - local git_options= format="--format %(refname:short)" - [[ -n $1 ]] && git_options="--git-dir=$1/.git" - - git $git_options for-each-ref $format 'refs/remotes' | sed 's,.*/,,' - git $git_options for-each-ref $format 'refs/heads' -} - -_rpkg_package() -{ - repoquery -C --qf=%{sourcerpm} "$1*" 2>/dev/null | sort -u | sed -r 's/(-[^-]*){2}\.src\.rpm$//' -} - -# Local variables: -# mode: shell-script -# sh-basic-offset: 4 -# sh-indent-comment: t -# indent-tabs-mode: nil -# End: -# ex: ts=4 sw=4 et filetype=sh diff --git a/src/rpkg.conf b/src/rpkg.conf deleted file mode 100644 index 40b6f15..0000000 --- a/src/rpkg.conf +++ /dev/null @@ -1,11 +0,0 @@ -[rpkg] -lookaside = http://localhost/repo/pkgs -lookasidehash = md5 -lookaside_cgi = https://localhost/repo/pkgs/upload.cgi -gitbaseurl = ssh://%(user)s@localhost/%(module)s -anongiturl = git://localhost/%(module)s -branchre = f\d$|f\d\d$|el\d$|olpc\d$|master$ -kojiconfig = /etc/koji.conf -build_client = koji -clone_config = - bz.default-component %(module)s diff --git a/src/rpkg_man_page.py b/src/rpkg_man_page.py deleted file mode 100644 index dcc5c8f..0000000 --- a/src/rpkg_man_page.py +++ /dev/null @@ -1,158 +0,0 @@ -# Print a man page from the help texts. -# -# Copyright (C) 2011 Red Hat Inc. -# Author(s): Jesse Keating -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - - -import sys -import datetime - - -# We could substitute the "" in .TH with the rpkg version if we knew it -man_header = """\ -.\\" man page for rpkg -.TH rpkg 1 "%(today)s" "" "rpm\-packager" -.SH "NAME" -rpkg \- RPM Packaging utility -.SH "SYNOPSIS" -.B "rpkg" -[ -.I global_options -] -.I "command" -[ -.I command_options -] -[ -.I command_arguments -] -.br -.B "rpkg" -.B "help" -.br -.B "rpkg" -.I "command" -.B "\-\-help" -.SH "DESCRIPTION" -.B "rpkg" -is a script to interact with the RPM Packaging system. -""" - -man_footer = """\ -.SH "SEE ALSO" -.UR "https://fedorahosted.org/rpkg/" -.BR "https://fedorahosted.org/rpkg/" -""" - - -class ManFormatter(object): - - def __init__(self, man): - self.man = man - - def write(self, data): - for line in data.split('\n'): - self.man.write(' %s\n' % line) - - -def strip_usage(s): - """Strip "usage: " string from beginning of string if present""" - if s.startswith('usage: '): - return s.replace('usage: ', '', 1) - else: - return s - - -def man_constants(): - """Global constants for man file templates""" - today = datetime.date.today() - today_manstr = today.strftime('%Y\-%m\-%d') - return {'today': today_manstr} - - -def generate(parser, subparsers): - """\ - Generate the man page on stdout - - Given the argparse based parser and subparsers arguments, generate - the corresponding man page and write it to stdout. - """ - - # Not nice, but works: Redirect any print statement output to - # stderr to avoid clobbering the man page output on stdout. - man_file = sys.stdout - sys.stdout = sys.stderr - - mf = ManFormatter(man_file) - - choices = subparsers.choices - k = sorted(choices.keys()) - - man_file.write(man_header % man_constants()) - - helptext = parser.format_help() - helptext = strip_usage(helptext) - helptextsplit = helptext.split('\n') - helptextsplit = [line for line in helptextsplit - if not line.startswith(' -h, --help')] - - man_file.write('.SS "%s"\n' % ("Global Options",)) - - outflag = False - for line in helptextsplit: - if line == "optional arguments:": - outflag = True - elif line == "": - outflag = False - elif outflag: - man_file.write("%s\n" % line) - - help_texts = {} - for pa in subparsers._choices_actions: - help_texts[pa.dest] = getattr(pa, 'help', None) - - man_file.write('.SH "COMMAND OVERVIEW"\n') - - for command in k: - cmdparser = choices[command] - if not cmdparser.add_help: - continue - usage = cmdparser.format_usage() - usage = strip_usage(usage) - usage = ''.join(usage.split('\n')) - usage = ' '.join(usage.split()) - if help_texts[command]: - man_file.write('.TP\n.B "%s"\n%s\n' % (usage, help_texts[command])) - else: - man_file.write('.TP\n.B "%s"\n' % (usage)) - - man_file.write('.SH "COMMAND REFERENCE"\n') - for command in k: - cmdparser = choices[command] - if not cmdparser.add_help: - continue - - man_file.write('.SS "%s"\n' % cmdparser.prog) - - help = help_texts[command] - if help and not cmdparser.description: - if not help.endswith('.'): - help = "%s." % help - cmdparser.description = help - - h = cmdparser.format_help() - mf.write(h) - - man_file.write(man_footer) - - -if __name__ == '__main__': - import pyrpkg.cli - client = pyrpkg.cli.cliClient(name='rpkg', config=None) - generate(client.parser, client.subparsers) diff --git a/test/commands/__init__.py b/test/commands/__init__.py deleted file mode 100644 index 380dc78..0000000 --- a/test/commands/__init__.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import shutil -import subprocess -import sys -import tempfile -import unittest - - -class CommandTestCase(unittest.TestCase): - def setUp(self): - self.origin_dir = os.getcwd() - self.path = tempfile.mkdtemp(prefix='rpkg-tests.') - self.gitroot = os.path.join(self.path, 'gitroot') - - self.module = 'module1' - - self.anongiturl = 'file://%s/%%(module)s' % self.gitroot - self.branchre = r'master|rpkg-tests-.+' - self.quiet = False - - # TODO: Figure out how to handle this - self.lookaside = 'TODO' - self.lookasidehash = 'md5' - self.lookaside_cgi = 'TODO' - self.gitbaseurl = 'TODO' - self.kojiconfig = 'TODO' - self.build_client = 'TODO' - self.clone_config = ''' - bz.default-component %(module)s - sendemail.to %(module)s-owner@fedoraproject.org - ''' - self.user = 'TODO' - self.dist = 'TODO' - self.target = 'TODO' - - def tearDown(self): - os.chdir(self.origin_dir) - shutil.rmtree(self.path) - - def make_new_git(self, module, branches=None): - """Make a new git repo, so that tests can clone it - - This is not a test method. - """ - if branches is None: - branches = [] - - # Create a bare Git repository - moduledir = os.path.join(self.gitroot, module) - os.makedirs(moduledir) - subprocess.check_call(['git', 'init', '--bare'], cwd=moduledir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - # Clone it, and do the minimal Dist Git setup - cloneroot = os.path.join(self.path, 'clonedir') - os.makedirs(cloneroot) - subprocess.check_call(['git', 'clone', 'file://%s' % moduledir], - cwd=cloneroot, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - clonedir = os.path.join(cloneroot, module.split('/')[-1]) - open(os.path.join(clonedir, '.gitignore'), 'w').close() - open(os.path.join(clonedir, 'sources'), 'w').close() - subprocess.check_call(['git', 'add', '.gitignore', 'sources'], - cwd=clonedir, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - subprocess.check_call(['git', 'commit', '-m', - 'Initial setup of the repo'], cwd=clonedir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - subprocess.check_call(['git', 'push', 'origin', 'master'], - cwd=clonedir, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # Add the requested branches - for branch in branches: - subprocess.check_call(['git', 'branch', branch], cwd=clonedir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - subprocess.check_call(['git', 'push', 'origin', branch], - cwd=clonedir, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # Drop the clone - shutil.rmtree(cloneroot) - - def get_tags(self, gitdir): - result = [] - - tags = subprocess.Popen(['git', 'tag', '-n1'], cwd=gitdir, - stdout=subprocess.PIPE, - universal_newlines=True).communicate()[0] - - for line in tags.split('\n'): - if not line: - continue - - tokens = [x for x in line.split() if x] - result.append([tokens[0], ' '.join(tokens[1:])]) - - return result - - def hijack_stdout(self): - class cm(object): - def __enter__(self): - from six.moves import cStringIO as StringIO - - self.old_stdout = sys.stdout - self.out = StringIO() - sys.stdout = self.out - - return self.out - - def __exit__(self, *args): - sys.stdout.flush() - sys.stdout = self.old_stdout - - self.out.seek(0) - - return cm() diff --git a/test/commands/test_add_tag.py b/test/commands/test_add_tag.py deleted file mode 100644 index 6e13404..0000000 --- a/test/commands/test_add_tag.py +++ /dev/null @@ -1,162 +0,0 @@ -import os - -from . import CommandTestCase - - -class CommandAddTagTestCase(CommandTestCase): - def setUp(self): - super(CommandAddTagTestCase, self).setUp() - if 'GIT_EDITOR' in os.environ: - self.old_git_editor = os.environ['GIT_EDITOR'] - else: - self.old_git_editor = None - - def tearDown(self): - if self.old_git_editor is not None: - os.environ['GIT_EDITOR'] = self.old_git_editor - super(CommandAddTagTestCase, self).tearDown() - - def test_add_tag(self): - self.make_new_git(self.module) - - tag = 'v1.0' - message = 'This is a release' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - # `git tag` will call $EDITOR to ask the user to write a message - os.environ['GIT_EDITOR'] = ('/usr/bin/python -c "import sys; ' - 'open(sys.argv[1], \'w\').write(\'%s\')"' - % message) - - cmd.add_tag(tag) - - self.assertEqual(self.get_tags(moduledir), [[tag, message]]) - - def test_add_tag_with_message(self): - self.make_new_git(self.module) - - tag = 'v1.0' - message = 'This is a release' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - cmd.add_tag(tag, message=message) - - self.assertEqual(self.get_tags(moduledir), [[tag, message]]) - - def test_add_tag_with_message_from_file(self): - self.make_new_git(self.module) - - tag = 'v1.0' - message = 'This is a release' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - message_file = os.path.join(moduledir, 'tag_message') - - with open(message_file, 'w') as f: - f.write(message) - - cmd.add_tag(tag, file=message_file) - - self.assertEqual(self.get_tags(moduledir), [[tag, message]]) - - def test_add_tag_fails_with_existing(self): - self.make_new_git(self.module) - - tag = 'v1.0' - message = 'This is a release' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - cmd.add_tag(tag, message=message) - - # Now add the same tag again - def raises(): - cmd.add_tag(tag, message='No, THIS is a release') - - self.assertRaises(pyrpkg.rpkgError, raises) - - def test_add_tag_force_replace_existing(self): - self.make_new_git(self.module) - - tag = 'v1.0' - message = 'This is a release' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - cmd.add_tag(tag, message=message) - - # Now add the same tag again by force - newmessage = 'No, THIS is a release' - cmd.add_tag(tag, message=newmessage, force=True) - - self.assertEqual(self.get_tags(moduledir), [[tag, newmessage]]) - - def test_add_tag_many(self): - self.make_new_git(self.module) - - tags = [['v1.0', 'This is a release'], - ['v2.0', 'This is another release']] - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - for tag, message in tags: - cmd.add_tag(tag, message=message) - - self.assertEqual(self.get_tags(moduledir), tags) diff --git a/test/commands/test_check_repo.py b/test/commands/test_check_repo.py deleted file mode 100644 index 0ca24df..0000000 --- a/test/commands/test_check_repo.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -import shutil -import subprocess -import tempfile - -from pyrpkg.errors import rpkgError - -from . import CommandTestCase - - -class CheckRepoCase(CommandTestCase): - - def setUp(self): - super(CheckRepoCase, self).setUp() - self.dist = "master" - self.make_new_git(self.module) - moduledir = os.path.join(self.gitroot, self.module) - - self.altpath = tempfile.mkdtemp(prefix='rpkg-tests.') - self.clonedir = os.path.join(self.altpath, self.module) - subprocess.check_call(['git', 'clone', 'file://%s' % moduledir], - cwd=self.altpath, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - import pyrpkg - self.cmd = pyrpkg.Commands( - self.clonedir, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet - ) - - def tearDown(self): - super(CheckRepoCase, self).tearDown() - # Drop the clone - shutil.rmtree(self.altpath) - - def test_repo_is_dirty(self): - with open(os.path.join(self.clonedir, 'sources'), 'w') as fd: - fd.write("a") - - try: - self.cmd.check_repo(is_dirty=True, all_pushed=False) - except rpkgError as exception: - self.assertTrue("has uncommitted changes" in str(exception)) - else: - self.fail("Expected an rpkgError exception.") - - def test_repo_has_unpushed_changes(self): - with open(os.path.join(self.clonedir, 'sources'), 'w') as fd: - fd.write("a") - subprocess.check_call( - ['git', 'add', 'sources'], - cwd=self.clonedir - ) - subprocess.check_call( - ['git', 'commit', '-m', 'commit sources'], - cwd=self.clonedir, - ) - - try: - self.cmd.check_repo(is_dirty=False, all_pushed=True) - except rpkgError as exception: - self.assertTrue("There are unpushed changes in your repo" in - str(exception)) - else: - self.fail("Expected an rpkgError exception.") - - def test_repo_is_clean(self): - self.cmd.check_repo(is_dirty=True, all_pushed=False) - - def test_repo_has_everything_pushed(self): - self.cmd.check_repo(is_dirty=False, all_pushed=True) diff --git a/test/commands/test_clone.py b/test/commands/test_clone.py deleted file mode 100644 index 5541a9b..0000000 --- a/test/commands/test_clone.py +++ /dev/null @@ -1,157 +0,0 @@ -import os -import shutil -import tempfile - -import git - -from . import CommandTestCase - - -CLONE_CONFIG = ''' - bz.default-component %(module)s - sendemail.to %(module)s-owner@fedoraproject.org -''' - - -class CommandCloneTestCase(CommandTestCase): - def test_clone_anonymous(self): - self.make_new_git(self.module) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone_config = CLONE_CONFIG - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git'))) - confgit = git.Git(moduledir) - self.assertEqual(confgit.config('bz.default-component'), self.module) - self.assertEqual(confgit.config('sendemail.to'), - "%s-owner@fedoraproject.org" % self.module) - - def test_clone_anonymous_with_namespace(self): - self.module = 'rpms/module1' - self.make_new_git(self.module) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet, distgit_namespaced=True) - cmd.clone_config = CLONE_CONFIG - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, 'module1') - self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git'))) - confgit = git.Git(moduledir) - self.assertEqual(confgit.config('bz.default-component'), self.module) - self.assertEqual(confgit.config('sendemail.to'), - "%s-owner@fedoraproject.org" % self.module) - - def test_clone_anonymous_with_path(self): - self.make_new_git(self.module) - - altpath = tempfile.mkdtemp(prefix='rpkg-tests.') - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True, path=altpath) - - moduledir = os.path.join(altpath, self.module) - self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git'))) - - notmoduledir = os.path.join(self.path, self.module) - self.assertFalse(os.path.isdir(os.path.join(notmoduledir, '.git'))) - - shutil.rmtree(altpath) - - def test_clone_anonymous_with_branch(self): - self.make_new_git(self.module, - branches=['rpkg-tests-1', 'rpkg-tests-2']) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True, branch='rpkg-tests-1') - - with open(os.path.join( - self.path, self.module, '.git', 'HEAD')) as HEAD: - self.assertEqual(HEAD.read(), 'ref: refs/heads/rpkg-tests-1\n') - - def test_clone_anonymous_with_bare_dir(self): - self.make_new_git(self.module) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True, bare_dir='%s.git' % self.module) - - clonedir = os.path.join(self.path, '%s.git' % self.module) - self.assertTrue(os.path.isdir(clonedir)) - self.assertFalse(os.path.exists(os.path.join(clonedir, 'index'))) - - def test_clone_fails_with_both_branch_and_bare_dir(self): - self.make_new_git(self.module, - branches=['rpkg-tests-1', 'rpkg-tests-2']) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - - def raises(): - cmd.clone(self.module, anon=True, branch='rpkg-tests-1', - bare_dir='test.git') - self.assertRaises(pyrpkg.rpkgError, raises) - - def test_clone_into_dir(self): - self.make_new_git(self.module, - branches=['rpkg-tests-1', 'rpkg-tests-2']) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone( - self.module, anon=True, branch='rpkg-tests-1', target='new_clone') - - with open(os.path.join( - self.path, 'new_clone', '.git', 'HEAD')) as HEAD: - self.assertEqual(HEAD.read(), 'ref: refs/heads/rpkg-tests-1\n') - - def test_clone_into_dir_with_namespace(self): - self.module = 'rpms/module1' - self.make_new_git(self.module, - branches=['rpkg-tests-1', 'rpkg-tests-2']) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet, distgit_namespaced=True) - cmd.clone( - self.module, anon=True, branch='rpkg-tests-1', target='new_clone') - - with open(os.path.join( - self.path, 'new_clone', '.git', 'HEAD')) as HEAD: - self.assertEqual(HEAD.read(), 'ref: refs/heads/rpkg-tests-1\n') diff --git a/test/commands/test_delete_tag.py b/test/commands/test_delete_tag.py deleted file mode 100644 index 2141059..0000000 --- a/test/commands/test_delete_tag.py +++ /dev/null @@ -1,52 +0,0 @@ -import os - -from . import CommandTestCase - - -class CommandDeleteTagTestCase(CommandTestCase): - def test_delete_tag(self): - self.make_new_git(self.module) - - tag = 'v1.0' - message = 'This is a release' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - # First, add a tag - cmd.add_tag(tag, message=message) - self.assertEqual(self.get_tags(moduledir), [[tag, message]]) - - # Now delete it - cmd.delete_tag(tag) - tags = [t for (t, m) in self.get_tags(moduledir)] - self.assertFalse(tag in tags) - - def test_delete_tag_fails_inexistent(self): - self.make_new_git(self.module) - - tag = 'v1.0' - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - # Try deleting an inexistent tag - def raises(): - cmd.delete_tag(tag) - self.assertRaises(pyrpkg.rpkgError, raises) diff --git a/test/commands/test_list_tag.py b/test/commands/test_list_tag.py deleted file mode 100644 index 8135f5e..0000000 --- a/test/commands/test_list_tag.py +++ /dev/null @@ -1,159 +0,0 @@ -import os - -from . import CommandTestCase - - -class CommandListTagTestCase(CommandTestCase): - def test_list_tag_no_tags(self): - self.make_new_git(self.module) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - with self.hijack_stdout() as out: - cmd.list_tag() - - self.assertEqual(out.read().strip(), '') - - def test_list_tag_many(self): - self.make_new_git(self.module) - - tags = [['v1.0', 'This is a release'], - ['v2.0', 'This is another release']] - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - for tag, message in tags: - cmd.add_tag(tag, message=message) - - with self.hijack_stdout() as out: - cmd.list_tag() - - result = out.read().strip().split('\n') - - self.assertEqual(result, [t for (t, m) in tags]) - - def test_list_tag_specific(self): - self.make_new_git(self.module) - - tags = [['v1.0', 'This is a release'], - ['v2.0', 'This is another release']] - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - for tag, message in tags: - cmd.add_tag(tag, message=message) - - with self.hijack_stdout() as out: - cmd.list_tag(tagname='v1.0') - - result = out.read().strip().split('\n') - - self.assertEqual(result, ['v1.0']) - - def test_list_tag_inexistent(self): - self.make_new_git(self.module) - - tags = [['v1.0', 'This is a release'], - ['v2.0', 'This is another release']] - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - for tag, message in tags: - cmd.add_tag(tag, message=message) - - with self.hijack_stdout() as out: - cmd.list_tag(tagname='v1.1') - - result = out.read().strip().split('\n') - - self.assertEqual(result, ['']) - - def test_list_tag_glob(self): - self.make_new_git(self.module) - - tags = [['v1.0', 'This is a release'], - ['v2.0', 'This is another release']] - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - for tag, message in tags: - cmd.add_tag(tag, message=message) - - with self.hijack_stdout() as out: - cmd.list_tag(tagname='v1*') - - result = out.read().strip().split('\n') - - self.assertEqual(result, ['v1.0']) - - def test_list_tag_wildcard(self): - self.make_new_git(self.module) - - tags = [['v1.0', 'This is a release'], - ['v2.0', 'This is another release']] - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - for tag, message in tags: - cmd.add_tag(tag, message=message) - - with self.hijack_stdout() as out: - cmd.list_tag(tagname='*') - - result = out.read().strip().split('\n') - - self.assertEqual(result, [t for (t, m) in tags]) diff --git a/test/commands/test_package_name.py b/test/commands/test_package_name.py deleted file mode 100644 index 23cdb69..0000000 --- a/test/commands/test_package_name.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -import six - -from . import CommandTestCase - - -class CommandPackageNameTestCase(CommandTestCase): - def test_name_is_not_unicode(self): - self.make_new_git(self.module) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone(self.module, anon=True) - - moduledir = os.path.join(self.path, self.module) - cmd.path = moduledir - - # pycurl can't handle unicode variable - # module_name needs to be a byte string - self.assertNotEqual(type(cmd.module_name), six.text_type) - self.assertEqual(type(cmd.module_name), six.binary_type) diff --git a/test/commands/test_patch.py b/test/commands/test_patch.py deleted file mode 100644 index 878871a..0000000 --- a/test/commands/test_patch.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- -import six - -from . import CommandTestCase - - -class CommandPatchTestCase(CommandTestCase): - def setUp(self): - super(CommandPatchTestCase, self).setUp() - self.text_ascii = "Lorem ipsum dolor sit amet, consectetur elit.\n" \ - "Sed vel enim nec tortor posuere sodales sit amet mauris.\n" \ - "Duis ipsum dui, consectetur pretium a, vestibulum.\n"\ - "Nunc vel consectetur libero. Aenean , metus quis posuere\n" \ - "vulputate, purus metus fringilla, sit amet interdum tellus\n" - self.text_utf8 = "ěšč\n" \ - "ščř\n" \ - "ýáí" - if six.PY3: - self.text_utf8 = self.text_utf8.encode("utf-8") - - def test_byte_offset_first_line(self): - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - line, offset = cmd._byte_offset_to_line_number(self.text_ascii, 10) - # 10 byte offset mean line 1 and character 11 - self.assertEqual(line, 1) - self.assertEqual(offset, 11) - - def test_byte_offset_next_line(self): - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - - line, offset = cmd._byte_offset_to_line_number(self.text_ascii, 46) - # 46 byte offset is first character on second line - self.assertEqual(line, 2) - self.assertEqual(offset, 1) - - def test_byte_offset_utf8(self): - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - text = self.text_utf8.decode('UTF-8', 'ignore') - line, offset = cmd._byte_offset_to_line_number(text, 9) - # 9 byte offset mean line 3 and second character - self.assertEqual(line, 3) - self.assertEqual(offset, 2) diff --git a/test/commands/test_push.py b/test/commands/test_push.py deleted file mode 100644 index 0c681f7..0000000 --- a/test/commands/test_push.py +++ /dev/null @@ -1,137 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import git - -from . import CommandTestCase - - -SPECFILE_TEMPLATE = """Name: test -Version: 1.0 -Release: 1.0 -Summary: test - -Group: Applications/System -License: GPLv2+ - -%s - -%%description -Test - -%%install -rm -f $RPM_BUILD_ROOT%%{_sysconfdir}/""" - -CLONE_CONFIG = ''' - bz.default-component %(module)s - sendemail.to %(module)s-owner@fedoraproject.org -''' - - -class CommandPushTestCase(CommandTestCase): - - def setUp(self): - # Tests within this case would change working directory. Changing back - # to original directory to avoid any potential problems. - self.original_dir = os.path.abspath(os.curdir) - super(CommandPushTestCase, self).setUp() - - def tearDown(self): - os.chdir(self.original_dir) - super(CommandPushTestCase, self).tearDown() - - def test_push_outside_repo(self): - """push from outside repo with --path option""" - - self.make_new_git(self.module) - - import pyrpkg - cmd = pyrpkg.Commands(self.path, self.lookaside, - self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - cmd.clone_config = CLONE_CONFIG - cmd.clone(self.module, anon=True) - cmd.path = os.path.join(self.path, self.module) - os.chdir(os.path.join(self.path, self.module)) - - spec_file = 'module.spec' - with open(spec_file, 'w') as f: - f.write(SPECFILE_TEMPLATE % '') - - cmd.repo.index.add([spec_file]) - cmd.repo.index.commit("add SPEC") - - # Now, change directory to parent and test the push - os.chdir(self.path) - cmd.push() - - -class TestPushWithPatches(CommandTestCase): - - def setUp(self): - super(TestPushWithPatches, self).setUp() - - self.make_new_git(self.module) - - import pyrpkg - self.cmd = pyrpkg.Commands(self.path, self.lookaside, - self.lookasidehash, - self.lookaside_cgi, self.gitbaseurl, - self.anongiturl, self.branchre, - self.kojiconfig, - self.build_client, self.user, self.dist, - self.target, self.quiet) - self.cmd.clone_config = CLONE_CONFIG - self.cmd.clone(self.module, anon=True) - self.cmd.path = os.path.join(self.path, self.module) - os.chdir(os.path.join(self.path, self.module)) - - # Track SPEC and a.patch in git - spec_file = 'module.spec' - with open(spec_file, 'w') as f: - f.write(SPECFILE_TEMPLATE % '''Patch0: a.patch -Patch1: b.path -Patch2: c.path -Patch3: d.path -''') - - for patch_file in ('a.patch', 'b.patch', 'c.patch', 'd.patch'): - with open(patch_file, 'w') as f: - f.write(patch_file) - - # Track c.patch in sources - from pyrpkg.sources import SourcesFile - sources_file = SourcesFile(self.cmd.sources_filename, - self.cmd.source_entry_type) - file_hash = self.cmd.lookasidecache.hash_file('c.patch') - sources_file.add_entry(self.cmd.lookasidehash, 'c.patch', file_hash) - sources_file.write() - - self.cmd.repo.index.add([spec_file, 'a.patch', 'sources']) - self.cmd.repo.index.commit('add SPEC and patches') - - def test_find_untracked_patches(self): - untracked_patches = self.cmd.find_untracked_patches() - untracked_patches.sort() - self.assertEqual(['b.patch', 'd.patch'], untracked_patches) - - def test_push_not_blocked_by_untracked_patches(self): - self.cmd.push() - - # Verify added files are pushed to origin - origin_repo_path = self.cmd.repo.git.config( - '--get', 'remote.origin.url').replace('file://', '') - origin_repo = git.Repo(origin_repo_path) - git_tree = origin_repo.head.commit.tree - self.assertTrue('a.patch' in git_tree) - self.assertTrue('b.patch' not in git_tree) - self.assertTrue('c.patch' not in git_tree) - self.assertTrue('d.patch' not in git_tree) - - sources_content = origin_repo.git.show('master:sources').strip() - with open('sources', 'r') as f: - expected_sources_content = f.read().strip() - self.assertEqual(expected_sources_content, sources_content) diff --git a/test/test_commands.py b/test/test_commands.py deleted file mode 100644 index 7267c89..0000000 --- a/test/test_commands.py +++ /dev/null @@ -1,432 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import shutil -import tempfile -import unittest -import subprocess - -import git -from mock import patch - -from pyrpkg import Commands -from pyrpkg import rpkgError - -# Following global variables are used to construct Commands for tests in this -# module. Only for testing purpose, and they are not going to be used for -# hitting real services. -lookaside = 'http://dist-git-qa.server/repo/pkgs' -lookaside_cgi = 'http://dist-git-qa.server/lookaside/upload.cgi' -gitbaseurl = 'ssh://%(user)s@dist-git-qa.server/rpms/%(module)s' -anongiturl = 'git://dist-git-qa.server/rpms/%(module)s' -lookasidehash = 'md5' -branchre = 'rhel' -kojiconfig = '/etc/koji.conf.d/brewstage.conf' -build_client = 'brew-stage' - -spec_file = ''' -Summary: Dummy summary -Name: docpkg -Version: 1.2 -Release: 2 -License: GPL -Group: Applications/Productivity -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -%description -This is a dummy description. -%prep -%build -%clean -rm -rf $$RPM_BUILD_ROOT -%install -rm -rf $RPM_BUILD_ROOT -mkdir $RPM_BUILD_ROOT -%files -%changelog -* Thu Apr 21 2006 Chenxiong Qi - 1.2-2 -- Initial version -''' - - -def run(cmd, **kwargs): - returncode = subprocess.call(cmd, **kwargs) - if returncode != 0: - raise RuntimeError('Command fails. Command: %s. Return code %d' % ( - ' '.join(cmd), returncode)) - - -class CommandTestCase(unittest.TestCase): - - def setUp(self): - # create a base repo - self.repo_path = tempfile.mkdtemp(prefix='rpkg-commands-tests-') - - # Add spec file to this repo and commit - spec_file_path = os.path.join(self.repo_path, 'package.spec') - with open(spec_file_path, 'w') as f: - f.write(spec_file) - - git_cmds = [ - ['git', 'init'], - ['git', 'add', spec_file_path], - ['git', 'config', 'user.email', 'cqi@redhat.com'], - ['git', 'config', 'user.name', 'Chenxiong Qi'], - ['git', 'commit', '-m', '"initial commit"'], - ['git', 'branch', 'eng-rhel-6'], - ['git', 'branch', 'eng-rhel-6.5'], - ['git', 'branch', 'eng-rhel-7'], - ] - for cmd in git_cmds: - run(cmd, cwd=self.repo_path) - - # Clone the repo - self.cloned_repo_path = tempfile.mkdtemp(prefix='rpkg-commands-tests-cloned-') - git_cmds = [ - ['git', 'clone', self.repo_path, self.cloned_repo_path], - ['git', 'branch', '--track', 'eng-rhel-6', 'origin/eng-rhel-6'], - ['git', 'branch', '--track', 'eng-rhel-6.5', 'origin/eng-rhel-6.5'], - ['git', 'branch', '--track', 'eng-rhel-7', 'origin/eng-rhel-7'], - ] - for cmd in git_cmds: - run(cmd, cwd=self.cloned_repo_path) - - def tearDown(self): - shutil.rmtree(self.repo_path) - shutil.rmtree(self.cloned_repo_path) - - def make_commands(self, path=None, user=None, dist=None, target=None, quiet=None): - """Helper method for creating Commands object for test cases - - This is where you should extend to add more features to support - additional requirements from other Commands specific test cases. - - Some tests need customize one of user, dist, target, and quiet options - when creating an instance of Commands. Keyword arguments user, dist, - target, and quiet here is for this purpose. - - :param str path: path to repository where this Commands will work on - top of - :param str user: user passed to --user option - :param str dist: dist passed to --dist option - :param str target: target passed to --target option - :param str quiet: quiet passed to --quiet option - """ - _repo_path = path if path else self.cloned_repo_path - return Commands(_repo_path, - lookaside, lookasidehash, lookaside_cgi, - gitbaseurl, anongiturl, - branchre, - kojiconfig, build_client, - user=user, dist=dist, target=target, quiet=quiet) - - def checkout_branch(self, repo, branch_name): - """Checkout to a local branch - - :param git.Repo repo: `git.Repo` instance represents a git repository - that current code works on top of. - :param str branch_name: name of local branch to checkout - """ - heads = [head for head in repo.heads if head.name == branch_name] - assert len(heads) > 0, \ - 'Repo must have a local branch named {} that ' \ - 'is for running tests. But now, it does not exist. Please check ' \ - 'if the repo is correct.'.format(branch_name) - - heads[0].checkout() - - def create_branch(self, repo, branch_name): - repo.git.branch(branch_name) - - def make_a_dummy_commit(self, repo): - filename = os.path.join(repo.working_dir, 'document.txt') - with open(filename, 'a+') as f: - f.write('Hello rpkg') - repo.index.add([filename]) - repo.index.commit('update document') - - -def mock_load_rpmdefines(self): - """Mock Commands.load_rpmdefines by setting empty list to _rpmdefines - - :param Commands self: load_rpmdefines is an instance method of Commands, - self is the instance whish is calling this method. - """ - self._rpmdefines = [] - - -def mock_load_spec(fake_spec): - """Return a mocked load_spec method that sets a fake spec to Commands - - :param str fake_spec: an arbitrary string representing a fake spec - file. What value is passed to fake_spec depends on the test purpose - completely. - """ - def mocked_load_spec(self): - """Mocked load_spec to set fake spec to an instance of Commands - - :param Commands self: load_spec is an instance method of Commands, self - is the instance which is calling this method. - """ - self._spec = fake_spec - return mocked_load_spec - - -def mock_load_branch_merge(fake_branch_merge): - """Return a mocked load_branch_merge method - - The mocked method sets a fake branch name to _branch_merge. - - :param str fake_branch_merge: an arbitrary string representing a fake - branch name. What value should be passed to fake_branch_merge depends on - the test purpose completely. - """ - def mocked_method(self): - """ - Mocked load_branch_merge to set fake branch name to an instance of - Commands. - - :param Commands self: load_branch_merge is an instance method of - Commands, so self is the instance which is calling this method. - """ - self._branch_merge = fake_branch_merge - return mocked_method - - -class LoadNameVerRelTest(CommandTestCase): - """Test case for Commands.load_nameverrel""" - - def setUp(self): - super(LoadNameVerRelTest, self).setUp() - self.cmd = self.make_commands() - self.checkout_branch(self.cmd.repo, 'eng-rhel-6') - - def test_load_from_spec(self): - """Ensure name, version, release can be loaded from a valid SPEC""" - self.cmd.load_nameverrel() - self.assertEqual('docpkg', self.cmd._module_name_spec) - self.assertEqual('0', self.cmd._epoch) - self.assertEqual('1.2', self.cmd._ver) - self.assertEqual('2', self.cmd._rel) - - def test_load_spec_where_path_contains_space(self): - """Ensure load_nameverrel works with a repo whose path contains space - - This test aims to test the space appearing in path does not break rpm - command execution. - - For this test purpose, firstly, original repo has to be cloned to a - new place which has a name containing arbitrary spaces. - """ - cloned_repo_dir = '/tmp/rpkg test cloned repo' - if os.path.exists(cloned_repo_dir): - shutil.rmtree(cloned_repo_dir) - cloned_repo = self.cmd.repo.clone(cloned_repo_dir) - - # Switching to branch eng-rhel-6 explicitly is required by running this - # on RHEL6/7 because an old version of git is available in the - # repo. - # The failure reason is, old version of git makes the master as the - # active branch in cloned repository, whatever the current active - # branch is in the remote repository. - # As of fixing this, I ran test on Fedora 23 with git 2.5.5, and test - # fails on RHEL7 with git 1.8.3.1 - cloned_repo.git.checkout('eng-rhel-6') - - cmd = self.make_commands(path=cloned_repo_dir) - - cmd.load_nameverrel() - self.assertEqual('docpkg', cmd._module_name_spec) - self.assertEqual('0', cmd._epoch) - self.assertEqual('1.2', cmd._ver) - self.assertEqual('2', cmd._rel) - - @patch('pyrpkg.Commands.load_rpmdefines', new=mock_load_rpmdefines) - @patch('pyrpkg.Commands.load_spec', - new=mock_load_spec('unknown-rpm-option a-nonexistent-package.spec')) - def test_load_when_rpm_fails(self): - """Ensure rpkgError is raised when rpm command fails - - Commands.load_spec is mocked to help generate an incorrect rpm command - line to cause the error that this test expects. - - Test test does not care about what rpm defines are retrieved from - repository, so setting an empty list to Commands._rpmdefines is safe - and enough. - """ - self.assertRaises(rpkgError, self.cmd.load_nameverrel) - - -class LoadBranchMergeTest(CommandTestCase): - """Test case for testing Commands.load_branch_merge""" - - def setUp(self): - super(LoadBranchMergeTest, self).setUp() - - self.cmd = self.make_commands() - - def test_load_branch_merge_from_eng_rhel_6(self): - self.checkout_branch(self.cmd.repo, 'eng-rhel-6') - self.cmd.load_branch_merge() - self.assertEqual(self.cmd._branch_merge, 'eng-rhel-6') - - def test_load_branch_merge_from_eng_rhel_6_5(self): - """ - Ensure load_branch_merge can work well against a more special branch - eng-rhel-6.5 - """ - self.checkout_branch(self.cmd.repo, 'eng-rhel-6.5') - self.cmd.load_branch_merge() - self.assertEqual(self.cmd._branch_merge, 'eng-rhel-6.5') - - def test_load_branch_merge_from_not_remote_merge_branch(self): - """Ensure load_branch_merge fails against local-branch - - A new local branch named local-branch is created for this test, loading - branch merge from this local branch should fail because there is no - configuration item branch.local-branch.merge. - """ - self.create_branch(self.cmd.repo, 'local-branch') - self.checkout_branch(self.cmd.repo, 'local-branch') - try: - self.cmd.load_branch_merge() - except rpkgError as e: - self.assertEqual('Unable to find remote branch. Use --dist', str(e)) - else: - self.fail("It's expected to raise rpkgError, but not.") - - def test_load_branch_merge_using_dist_option(self): - """Ensure load_branch_merge uses dist specified via --dist - - Switch to eng-rhel-6 branch, that is valid for load_branch_merge and to - see if load_branch_merge still uses dist rather than such a valid - branch. - """ - self.checkout_branch(self.cmd.repo, 'eng-rhel-6') - - cmd = self.make_commands(dist='branch_merge') - cmd.load_branch_merge() - self.assertEqual('branch_merge', cmd._branch_merge) - - -class LoadRPMDefinesTest(CommandTestCase): - """Test case for Commands.load_rpmdefines""" - - def setUp(self): - super(LoadRPMDefinesTest, self).setUp() - self.cmd = self.make_commands() - - def assert_loaded_rpmdefines(self, branch_name, expected_defines): - self.checkout_branch(self.cmd.repo, branch_name) - - self.cmd.load_rpmdefines() - self.assertTrue(self.cmd._rpmdefines) - - # Convert defines into dict for assertion conveniently. The dict - # contains mapping from variable name to value. For example, - # { - # '_sourcedir': '/path/to/src-dir', - # '_specdir': '/path/to/spec', - # '_builddir': '/path/to/build-dir', - # '_srcrpmdir': '/path/to/srcrpm-dir', - # 'dist': 'el7' - # } - defines = dict([item.split(' ') for item in ( - define.replace("'", '').split(' ', 1)[1] for - define in self.cmd._rpmdefines)]) - - for var, val in expected_defines.items(): - self.assertTrue(var in defines) - self.assertEqual(val, defines[var]) - - def test_load_rpmdefines_from_eng_rhel_6(self): - """Run load_rpmdefines against branch eng-rhel-6""" - expected_rpmdefines = { - '_sourcedir': self.cloned_repo_path, - '_specdir': self.cloned_repo_path, - '_builddir': self.cloned_repo_path, - '_srcrpmdir': self.cloned_repo_path, - '_rpmdir': self.cloned_repo_path, - 'dist': u'.el6', - 'rhel': u'6', - 'el6': u'1', - } - self.assert_loaded_rpmdefines('eng-rhel-6', expected_rpmdefines) - - def test_load_rpmdefines_from_eng_rhel_6_5(self): - """Run load_rpmdefines against branch eng-rhel-6.5 - - Working on a different branch name is the only difference from test - method test_load_rpmdefines_from_eng_rhel_6. - """ - expected_rpmdefines = { - '_sourcedir': self.cloned_repo_path, - '_specdir': self.cloned_repo_path, - '_builddir': self.cloned_repo_path, - '_srcrpmdir': self.cloned_repo_path, - '_rpmdir': self.cloned_repo_path, - 'dist': u'.el6_5', - 'rhel': u'6', - 'el6_5': u'1', - } - self.assert_loaded_rpmdefines('eng-rhel-6.5', expected_rpmdefines) - - @patch('pyrpkg.Commands.load_branch_merge', - new=mock_load_branch_merge('invalid-branch-name')) - def test_load_rpmdefines_against_invalid_branch(self): - """Ensure load_rpmdefines if active branch name is invalid - - This test requires an invalid branch name even if - Commands.load_branch_merge is able to get it from current active - branch. So, I only care about the value returned from method - load_branch_merge, and just mock it and let it return the value this - test requires. - """ - self.assertRaises(rpkgError, self.cmd.load_rpmdefines) - - -class CheckRepoWithOrWithoutDistOptionCase(CommandTestCase): - """Check whether there are unpushed changes with or without specified dist - - Ensure check_repo works in a correct way to check if there are unpushed - changes, and this should not be affected by specified dist or not. - Bug 1169663 describes a concrete use case and this test case is designed - as what that bug describs. - """ - - def setUp(self): - super(CheckRepoWithOrWithoutDistOptionCase, self).setUp() - - private_branch = 'private-dev-branch' - origin_repo = git.Repo(self.repo_path) - origin_repo.git.checkout('master') - origin_repo.git.branch(private_branch) - self.make_a_dummy_commit(origin_repo) - - cloned_repo = git.Repo(self.cloned_repo_path) - cloned_repo.git.pull() - cloned_repo.git.checkout('-b', private_branch, 'origin/%s' % private_branch) - for i in xrange(3): - self.make_a_dummy_commit(cloned_repo) - cloned_repo.git.push() - - def test_check_repo_with_specificed_dist(self): - cmd = self.make_commands(self.cloned_repo_path, dist='eng-rhel-6') - try: - cmd.check_repo() - except rpkgError as e: - if 'There are unpushed changes in your repo' in e: - self.fail('There are unpushed changes in your repo. This ' - 'should not happen. Something must be going wrong.') - - self.fail('Should not fail. Something must be going wrong.') - - def test_check_repo_without_specificed_dist(self): - cmd = self.make_commands(self.cloned_repo_path) - try: - cmd.check_repo() - except rpkgError as e: - if 'There are unpushed changes in your repo' in e: - self.fail('There are unpushed changes in your repo. This ' - 'should not happen. Something must be going wrong.') - - self.fail('Should not fail. Something must be going wrong.') diff --git a/test/test_gitgnore.py b/test/test_gitgnore.py deleted file mode 100644 index 2fc7c17..0000000 --- a/test/test_gitgnore.py +++ /dev/null @@ -1,109 +0,0 @@ -import os -import shutil -import tempfile -import unittest - - -class GitIgnoreTestCase(unittest.TestCase): - def setUp(self): - self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') - - def tearDown(self): - shutil.rmtree(self.workdir) - - def test_add_existing_line(self): - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) - gi.add('a new line') - self.assertTrue(gi.modified) - - # Cheat a bit for unit tests - gi.modified = False - - gi.add('a new line') - self.assertFalse(gi.modified) - - gi.add('*') - self.assertTrue(gi.modified) - - # Cheat a bit for unit tests - gi.modified = False - - gi.add('something different') - self.assertFalse(gi.modified) - - def test_match_empty(self): - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) - self.assertFalse(gi.match('this does not exist')) - - # The empty string could match an empty file, but we don't want it to - self.assertFalse(gi.match('')) - - def test_match_line_from_existing_file(self): - gi_path = os.path.join(self.workdir, 'gitignore') - - with open(gi_path, 'w') as f: - f.write('this line exists\n') - - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(gi_path) - self.assertTrue(gi.match('this line exists')) - self.assertTrue(gi.match('this line exists\n')) - self.assertTrue(gi.match('/this line exists')) - self.assertTrue(gi.match('/this line exists\n')) - - self.assertFalse(gi.match('but this line does not')) - - def test_match_unwritten_line(self): - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) - gi.add('here is a new line') - - self.assertTrue(gi.modified) - self.assertTrue(gi.match('here is a new line')) - - def test_match_glob(self): - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) - gi.add('*') - - self.assertTrue(gi.match('Surely this is matched by a wildcard?')) - - def test_write_new_file(self): - gi_path = os.path.join(self.workdir, 'gitignore') - - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(gi_path) - gi.add('here is a new line') - gi.write() - - self.assertFalse(gi.modified) - - with open(gi_path) as f: - self.assertEqual(f.read(), 'here is a new line\n') - - def test_write_append_to_existing_file(self): - gi_path = os.path.join(self.workdir, 'gitignore') - - lines = ('this line exists', 'here is a new line') - - with open(gi_path, 'w') as f: - f.write(lines[0]) - - from pyrpkg.gitignore import GitIgnore - - gi = GitIgnore(gi_path) - gi.add(lines[1]) - gi.write() - - self.assertFalse(gi.modified) - - with open(gi_path) as f: - self.assertEqual(f.read(), '%s\n' % '\n'.join(lines)) diff --git a/test/test_lookaside.py b/test/test_lookaside.py deleted file mode 100644 index 7509e61..0000000 --- a/test/test_lookaside.py +++ /dev/null @@ -1,586 +0,0 @@ -# Copyright (c) 2015 - Red Hat Inc. -# -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation; either version 2 of the License, or (at your -# option) any later version. See http://www.gnu.org/copyleft/gpl.html for -# the full text of the license. - - -import hashlib -import os -import shutil -import tempfile -import unittest - -import mock -import pycurl - -from pyrpkg.lookaside import CGILookasideCache -from pyrpkg.errors import DownloadError, InvalidHashType, UploadError - - -class CGILookasideCacheTestCase(unittest.TestCase): - def setUp(self): - self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') - self.filename = os.path.join(self.workdir, self._testMethodName) - - def tearDown(self): - shutil.rmtree(self.workdir) - - def test_hash_file(self): - lc = CGILookasideCache('sha512', '_', '_') - - with open(self.filename, 'w') as f: - f.write('something') - - result = lc.hash_file(self.filename, 'md5') - self.assertEqual(result, '437b930db84b8079c2dd804a71936b5f') - - result = lc.hash_file(self.filename) - self.assertEqual(result, '983d43ddff6da90f6a5d3b6172446a1ffe228b803fe64fdd5dcfab5646078a896851fe82f623c9d6e5654b3d2f363a04ec17cfb62b607437a9c7c132d511e522') # nopep8 - - def test_hash_file_invalid_hash_type(self): - lc = CGILookasideCache('sha512', '_', '_') - self.assertRaises(InvalidHashType, lc.hash_file, '_', 'sha42') - - def test_hash_file_empty(self): - lc = CGILookasideCache('sha512', '_', '_') - - with open(self.filename, 'w') as f: - f.write('') - - result = lc.hash_file(self.filename, 'md5') - self.assertEqual(result, 'd41d8cd98f00b204e9800998ecf8427e') - - def test_file_is_valid(self): - lc = CGILookasideCache('md5', '_', '_') - - with open(self.filename, 'w') as f: - f.write('something') - - self.assertTrue(lc.file_is_valid(self.filename, - '437b930db84b8079c2dd804a71936b5f')) - self.assertFalse(lc.file_is_valid(self.filename, 'not the right hash', - hashtype='sha512')) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_download(self, mock_curl): - def mock_getinfo(info): - return 200 if info == pycurl.RESPONSE_CODE else 0 - - def mock_perform(): - with open(self.filename, 'rb') as f: - curlopts[pycurl.WRITEDATA].write(f.read()) - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.side_effect = mock_getinfo - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - with open(self.filename, 'wb') as f: - f.write(b'content') - - name = 'pyrpkg' - filename = 'pyrpkg-0.0.tar.xz' - hash = hashlib.sha512(b'content').hexdigest() - outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') - full_url = 'http://example.com/%s/%s/%s/%s' % (name, filename, hash, - filename) - - lc = CGILookasideCache('sha512', 'http://example.com', '_') - lc.download(name, filename, hash, outfile, hashtype='sha512') - self.assertEqual(curl.perform.call_count, 1) - self.assertEqual(curlopts[pycurl.URL], full_url) - self.assertEqual(os.path.getmtime(outfile), 0) - - with open(outfile) as f: - self.assertEqual(f.read(), 'content') - - # Try a second time - lc.download(name, filename, hash, outfile) - self.assertEqual(curl.perform.call_count, 1) - - # Try a third time - os.remove(outfile) - lc.download(name, filename, hash, outfile) - self.assertEqual(curl.perform.call_count, 2) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_download_kwargs(self, mock_curl): - def mock_getinfo(info): - return 200 if info == pycurl.RESPONSE_CODE else 0 - - def mock_perform(): - with open(self.filename, 'rb') as f: - curlopts[pycurl.WRITEDATA].write(f.read()) - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.side_effect = mock_getinfo - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - with open(self.filename, 'wb') as f: - f.write(b'content') - - name = 'pyrpkg' - filename = 'pyrpkg-0.0.tar.xz' - branch = 'f22' - hash = hashlib.sha512(b'content').hexdigest() - outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') - - path = '%(name)s/%(filename)s/%(branch)s/%(hashtype)s/%(hash)s' - full_url = 'http://example.com/%s' % ( - path % {'name': name, 'filename': filename, 'branch': branch, - 'hashtype': 'sha512', 'hash': hash}) - - lc = CGILookasideCache('sha512', 'http://example.com', '_') - - # Modify the download path, to try arbitrary kwargs - lc.download_path = path - - lc.download(name, filename, hash, outfile, hashtype='sha512', - branch=branch) - self.assertEqual(curl.perform.call_count, 1) - self.assertEqual(curlopts[pycurl.URL], full_url) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_download_corrupted(self, mock_curl): - def mock_getinfo(info): - return 200 if info == pycurl.RESPONSE_CODE else 0 - - def mock_perform(): - with open(self.filename) as f: - curlopts[pycurl.WRITEDATA].write(f.read()) - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.side_effect = mock_getinfo - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - with open(self.filename, 'wb') as f: - f.write(b'content') - - hash = "not the right hash" - outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') - - lc = CGILookasideCache('sha512', 'http://example.com', '_') - self.assertRaises(DownloadError, lc.download, 'pyrpkg', - 'pyrpkg-0.0.tar.xz', hash, outfile) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_download_failed(self, mock_curl): - curl = mock_curl.return_value - curl.perform.side_effect = Exception( - 'Could not resolve host: example.com') - - with open(self.filename, 'wb') as f: - f.write(b'content') - - hash = hashlib.sha512(b'content').hexdigest() - outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') - - lc = CGILookasideCache('sha512', 'http://example.com', '_') - self.assertRaises(DownloadError, lc.download, 'pyrpkg', - 'pyrpkg-0.0.tar.xz', hash, outfile) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_download_failed_status_code(self, mock_curl): - def mock_getinfo(info): - return 500 if info == pycurl.RESPONSE_CODE else 0 - - def mock_perform(): - with open(self.filename) as f: - curlopts[pycurl.WRITEDATA].write(f.read()) - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.side_effect = mock_getinfo - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - with open(self.filename, 'wb') as f: - f.write(b'content') - - hash = hashlib.sha512(b'content').hexdigest() - outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') - - lc = CGILookasideCache('sha512', 'http://example.com', '_') - self.assertRaises(DownloadError, lc.download, 'pyrpkg', - 'pyrpkg-0.0.tar.xz', hash, outfile) - - @mock.patch('pyrpkg.lookaside.sys.stdout') - def test_print_download_progress(self, mock_stdout): - def mock_write(msg): - written_lines.append(msg) - - written_lines = [] - expected_lines = [ - '\r################## 25.0%', # nopep8 - '\r#################################### 50.0%', # nopep8 - '\r###################################################### 75.0%', # nopep8 - '\r######################################################################## 100.0%', # nopep8 - ] - - mock_stdout.write.side_effect = mock_write - - lc = CGILookasideCache('_', '_', '_') - lc.print_progress(2000.0, 500.0, 0.0, 0.0) - self.assertEqual(mock_stdout.write.call_count, 1) - self.assertEqual(len(written_lines), 1) - - lc.print_progress(2000.0, 1000.0, 0.0, 0.0) - self.assertEqual(mock_stdout.write.call_count, 2) - self.assertEqual(len(written_lines), 2) - - lc.print_progress(2000.0, 1500.0, 0.0, 0.0) - self.assertEqual(mock_stdout.write.call_count, 3) - self.assertEqual(len(written_lines), 3) - - lc.print_progress(2000.0, 2000.0, 0.0, 0.0) - self.assertEqual(mock_stdout.write.call_count, 4) - self.assertEqual(len(written_lines), 4) - - self.assertEqual(written_lines, expected_lines) - - @mock.patch('pyrpkg.lookaside.sys.stdout') - def test_print_upload_progress(self, mock_stdout): - def mock_write(msg): - written_lines.append(msg) - - written_lines = [] - expected_lines = [ - '\r################## 25.0%', # nopep8 - '\r#################################### 50.0%', # nopep8 - '\r###################################################### 75.0%', # nopep8 - '\r######################################################################## 100.0%', # nopep8 - ] - - mock_stdout.write.side_effect = mock_write - - lc = CGILookasideCache('_', '_', '_') - lc.print_progress(0.0, 0.0, 2000.0, 500.0) - self.assertEqual(mock_stdout.write.call_count, 1) - self.assertEqual(len(written_lines), 1) - - lc.print_progress(0.0, 0.0, 2000.0, 1000.0) - self.assertEqual(mock_stdout.write.call_count, 2) - self.assertEqual(len(written_lines), 2) - - lc.print_progress(0.0, 0.0, 2000.0, 1500.0) - self.assertEqual(mock_stdout.write.call_count, 3) - self.assertEqual(len(written_lines), 3) - - lc.print_progress(0.0, 0.0, 2000.0, 2000.0) - self.assertEqual(mock_stdout.write.call_count, 4) - self.assertEqual(len(written_lines), 4) - - self.assertEqual(written_lines, expected_lines) - - @mock.patch('pyrpkg.lookaside.sys.stdout') - def test_print_no_progress(self, mock_stdout): - def mock_write(msg): - written_lines.append(msg) - - written_lines = [] - - mock_stdout.write.side_effect = mock_write - - lc = CGILookasideCache('_', '_', '_') - lc.print_progress(0.0, 0.0, 0.0, 0.0) - self.assertEqual(len(written_lines), 0) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_exists(self, mock_curl): - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION](b'Available') - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - lc = CGILookasideCache('_', '_', '_') - exists = lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') - self.assertTrue(exists) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_does_not_exist(self, mock_curl): - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION](b'Missing') - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - lc = CGILookasideCache('_', '_', '_') - exists = lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') - self.assertFalse(exists) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_exists_with_custom_certs(self, mock_curl): - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION](b'Available') - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - client_cert = os.path.join(self.workdir, 'my-client-cert.cert') - with open(client_cert, 'w'): - pass - - ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') - with open(ca_cert, 'w'): - pass - - lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, - ca_cert=ca_cert) - lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') - self.assertEqual(curlopts[pycurl.SSLCERT], client_cert) - self.assertEqual(curlopts[pycurl.CAINFO], ca_cert) - - @mock.patch('pyrpkg.lookaside.logging.getLogger') - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_exists_missing_custom_certs(self, mock_curl, - mock_logger): - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION](b'Available') - - def mock_setopt(opt, value): - curlopts[opt] = value - - def mock_warn(msg): - warn_messages.append(msg) - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - warn_messages = [] - log = mock_logger.return_value - log.warning.side_effect = mock_warn - - client_cert = os.path.join(self.workdir, 'my-client-cert.cert') - ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') - - lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, - ca_cert=ca_cert) - lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') - self.assertTrue(pycurl.SSLCERT not in curlopts) - self.assertTrue(pycurl.CAINFO not in curlopts) - self.assertEqual(len(warn_messages), 2) - self.assertTrue('Missing certificate: ' in warn_messages[0]) - self.assertTrue('Missing certificate: ' in warn_messages[1]) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_exists_check_failed(self, mock_curl): - curl = mock_curl.return_value - curl.perform.side_effect = Exception( - 'Could not resolve host: example.com') - - lc = CGILookasideCache('_', '_', '_') - self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg', - 'pyrpkg-0.tar.xz', 'thehash') - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_exists_check_failed_status_code(self, mock_curl): - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION](b'Available') - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 500 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - lc = CGILookasideCache('_', '_', '_') - self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg', - 'pyrpkg-0.0.tar.xz', 'thehash') - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_remote_file_exists_check_unexpected_error(self, mock_curl): - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION]('Something unexpected') - - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - lc = CGILookasideCache('_', '_', '_') - self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg', - 'pyrpkg-0.tar.xz', 'thehash') - - @mock.patch('pyrpkg.lookaside.logging.getLogger') - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_upload(self, mock_curl, mock_logger): - def mock_setopt(opt, value): - curlopts[opt] = value - - def mock_perform(): - curlopts[pycurl.WRITEFUNCTION](b'Some output') - - def mock_debug(msg): - debug_messages.append(msg) - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.perform.side_effect = mock_perform - curl.setopt.side_effect = mock_setopt - - debug_messages = [] - log = mock_logger.return_value - log.debug.side_effect = mock_debug - - lc = CGILookasideCache('sha512', '_', '_') - - with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): - lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', 'thehash') - - self.assertTrue(pycurl.HTTPPOST in curlopts) - self.assertEqual(curlopts[pycurl.HTTPPOST], [ - ('name', 'pyrpkg'), ('sha512sum', 'thehash'), - ('file', (pycurl.FORM_FILE, 'pyrpkg-0.0.tar.xz'))]) - - self.assertEqual(debug_messages, [b'Some output']) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_upload_already_exists(self, mock_curl): - curl = mock_curl.return_value - - lc = CGILookasideCache('_', '_', '_') - hash = 'thehash' - - with mock.patch.object(lc, 'remote_file_exists', lambda *x: True): - lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', hash) - - self.assertEqual(curl.perform.call_count, 0) - self.assertEqual(curl.setopt.call_count, 0) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_upload_with_custom_certs(self, mock_curl): - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.setopt.side_effect = mock_setopt - - client_cert = os.path.join(self.workdir, 'my-client-cert.cert') - with open(client_cert, 'w'): - pass - - ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') - with open(ca_cert, 'w'): - pass - - lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, - ca_cert=ca_cert) - - with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): - lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', 'thehash') - - self.assertEqual(curlopts[pycurl.SSLCERT], client_cert) - self.assertEqual(curlopts[pycurl.CAINFO], ca_cert) - - @mock.patch('pyrpkg.lookaside.logging.getLogger') - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_upload_missing_custom_certs(self, mock_curl, mock_logger): - def mock_setopt(opt, value): - curlopts[opt] = value - - def mock_warn(msg): - warn_messages.append(msg) - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 200 - curl.setopt.side_effect = mock_setopt - - warn_messages = [] - log = mock_logger.return_value - log.warning.side_effect = mock_warn - - client_cert = os.path.join(self.workdir, 'my-client-cert.cert') - ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') - - lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, - ca_cert=ca_cert) - - with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): - lc.upload('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') - - self.assertTrue(pycurl.SSLCERT not in curlopts) - self.assertTrue(pycurl.CAINFO not in curlopts) - self.assertEqual(len(warn_messages), 2) - self.assertTrue('Missing certificate: ' in warn_messages[0]) - self.assertTrue('Missing certificate: ' in warn_messages[1]) - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_upload_failed(self, mock_curl): - curl = mock_curl.return_value - curl.perform.side_effect = Exception( - 'Could not resolve host: example.com') - - lc = CGILookasideCache('_', '_', '_') - - with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): - self.assertRaises(UploadError, lc.upload, 'pyrpkg', - 'pyrpkg-0.tar.xz', 'thehash') - - @mock.patch('pyrpkg.lookaside.pycurl.Curl') - def test_upload_failed_status_code(self, mock_curl): - def mock_setopt(opt, value): - curlopts[opt] = value - - curlopts = {} - curl = mock_curl.return_value - curl.getinfo.return_value = 500 - curl.setopt.side_effect = mock_setopt - - lc = CGILookasideCache('sha512', '_', '_') - - with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): - self.assertRaises(UploadError, lc.upload, 'pyrpkg', - 'pyrpkg-0.tar.xz', 'thehash') diff --git a/test/test_sources.py b/test/test_sources.py deleted file mode 100644 index fafbaf9..0000000 --- a/test/test_sources.py +++ /dev/null @@ -1,270 +0,0 @@ -import os -import shutil -import tempfile -import unittest - -from pyrpkg import sources - - -class SourceFileEntryTestCase(unittest.TestCase): - def test_entry(self): - e = sources.SourceFileEntry('md5', 'afile', 'ahash') - expected = 'ahash afile\n' - self.assertEqual(str(e), expected) - - def test_bsd_style_entry(self): - e = sources.BSDSourceFileEntry('md5', 'afile', 'ahash') - expected = 'MD5 (afile) = ahash\n' - self.assertEqual(str(e), expected) - - -class SourcesFileTestCase(unittest.TestCase): - def setUp(self): - self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') - self.sourcesfile = os.path.join(self.workdir, self._testMethodName) - - def tearDown(self): - shutil.rmtree(self.workdir) - - def test_parse_empty_line(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - entry = s.parse_line('') - self.assertTrue(entry is None) - - def test_parse_eol_line(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - entry = s.parse_line('\n') - self.assertTrue(entry is None) - - def test_parse_whitespace_line(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - entry = s.parse_line(' \n') - self.assertTrue(entry is None) - - def test_parse_old_style_line(self): - s = sources.SourcesFile(self.sourcesfile, 'old') - - line = 'ahash afile\n' - entry = s.parse_line(line) - - self.assertTrue(isinstance(entry, sources.SourceFileEntry)) - self.assertEqual(entry.hashtype, 'md5') - self.assertEqual(entry.hash, 'ahash') - self.assertEqual(entry.file, 'afile') - self.assertEqual(str(entry), line) - - def test_migrate_old_style_line(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - line = 'ahash afile\n' - newline = 'MD5 (afile) = ahash\n' - entry = s.parse_line(line) - - self.assertTrue(isinstance(entry, sources.SourceFileEntry)) - self.assertEqual(entry.hashtype, 'md5') - self.assertEqual(entry.hash, 'ahash') - self.assertEqual(entry.file, 'afile') - self.assertEqual(str(entry), newline) - - def test_parse_entry_line(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - line = 'MD5 (afile) = ahash\n' - entry = s.parse_line(line) - - self.assertTrue(isinstance(entry, sources.SourceFileEntry)) - self.assertEqual(entry.hashtype, 'md5') - self.assertEqual(entry.hash, 'ahash') - self.assertEqual(entry.file, 'afile') - self.assertEqual(str(entry), line) - - def test_parse_wrong_lines(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - lines = ['ahash', - 'ahash ', - 'ahash afile', - 'SHA512 (afile) = ahash garbage', - 'MD5 SHA512 (afile) = ahash', - ] - - for line in lines: - def raises(): - s.parse_line(line) - - self.assertRaises(sources.MalformedLineError, raises) - - def test_open_new_file(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - self.assertEqual(len(s.entries), 0) - - def test_open_empty_file(self): - with open(self.sourcesfile, 'w') as f: - f.write('') - - s = sources.SourcesFile(self.sourcesfile, 'bsd') - self.assertEqual(len(s.entries), 0) - - def test_open_existing_file_with_old_style_lines(self): - lines = ['ahash afile\n', 'anotherhash anotherfile\n'] - newlines = ['MD5 (afile) = ahash\n', - 'MD5 (anotherfile) = anotherhash\n'] - - with open(self.sourcesfile, 'w') as f: - for line in lines: - f.write(line) - - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - for i, entry in enumerate(s.entries): - self.assertTrue(isinstance(entry, sources.SourceFileEntry)) - self.assertEqual(str(entry), newlines[i]) - - def test_open_existing_file(self): - lines = ['MD5 (afile) = ahash\n', 'MD5 (anotherfile) = anotherhash\n'] - - with open(self.sourcesfile, 'w') as f: - for line in lines: - f.write(line) - - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - for i, entry in enumerate(s.entries): - self.assertTrue(isinstance(entry, sources.SourceFileEntry)) - self.assertEqual(str(entry), lines[i]) - - def test_open_existing_file_with_mixed_lines(self): - lines = ['ahash afile\n', - 'anotherhash anotherfile\n', - 'MD5 (thirdfile) = thirdhash\n', - ] - expected = [ - 'MD5 (afile) = ahash\n', - 'MD5 (anotherfile) = anotherhash\n', - 'MD5 (thirdfile) = thirdhash\n', - ] - - with open(self.sourcesfile, 'w') as f: - for line in lines: - f.write(line) - - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - for i, entry in enumerate(s.entries): - self.assertTrue(isinstance(entry, sources.SourceFileEntry)) - self.assertEqual(str(entry), expected[i]) - - def test_open_existing_file_with_identical_entries_old_and_new(self): - lines = ['ahash afile\n', - 'MD5 (afile) = ahash\n', - ] - - with open(self.sourcesfile, 'w') as f: - for line in lines: - f.write(line) - - s = sources.SourcesFile(self.sourcesfile, 'bsd') - - self.assertEqual(len(s.entries), 1) - self.assertEqual(s.entries[0].hashtype, 'md5') - self.assertEqual(s.entries[0].file, 'afile') - self.assertEqual(s.entries[0].hash, 'ahash') - self.assertEqual(str(s.entries[0]), lines[-1]) - - def test_open_existing_file_with_wrong_line(self): - line = 'some garbage here\n' - - with open(self.sourcesfile, 'w') as f: - f.write(line) - - def raises(): - sources.SourcesFile(self.sourcesfile, 'bsd') - - self.assertRaises(sources.MalformedLineError, raises) - - def test_add_entry(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - self.assertEqual(len(s.entries), 0) - - s.add_entry('md5', 'afile', 'ahash') - self.assertEqual(len(s.entries), 1) - self.assertEqual(str(s.entries[-1]), 'MD5 (afile) = ahash\n') - - s.add_entry('md5', 'anotherfile', 'anotherhash') - self.assertEqual(len(s.entries), 2) - self.assertEqual(str(s.entries[-1]), 'MD5 (anotherfile) = anotherhash\n') - - def test_add_entry_twice(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - self.assertEqual(len(s.entries), 0) - - s.add_entry('md5', 'afile', 'ahash') - self.assertEqual(len(s.entries), 1) - self.assertEqual(str(s.entries[-1]), 'MD5 (afile) = ahash\n') - - s.add_entry('md5', 'afile', 'ahash') - self.assertEqual(len(s.entries), 1) - - def test_add_entry_mixing_hashtypes(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - self.assertEqual(len(s.entries), 0) - - s.add_entry('md5', 'afile', 'ahash') - self.assertEqual(len(s.entries), 1) - self.assertEqual(str(s.entries[-1]), 'MD5 (afile) = ahash\n') - - def raises(): - s.add_entry('sha512', 'anotherfile', 'anotherhash') - - self.assertRaises(sources.HashtypeMixingError, raises) - - def test_write_new_file(self): - s = sources.SourcesFile(self.sourcesfile, 'bsd') - self.assertEqual(len(s.entries), 0) - - s.add_entry('md5', 'afile', 'ahash') - s.add_entry('md5', 'anotherfile', 'anotherhash') - s.write() - - with open(self.sourcesfile) as f: - lines = f.readlines() - - self.assertEqual(len(lines), 2) - self.assertEqual(lines[0], 'MD5 (afile) = ahash\n') - self.assertEqual(lines[1], 'MD5 (anotherfile) = anotherhash\n') - - def test_write_adding_a_line(self): - lines = ['ahash afile\n', 'anotherhash anotherfile\n'] - - with open(self.sourcesfile, 'w') as f: - for line in lines: - f.write(line) - - s = sources.SourcesFile(self.sourcesfile, 'bsd') - s.add_entry('md5', 'thirdfile', 'thirdhash') - s.write() - - with open(self.sourcesfile) as f: - lines = f.readlines() - - self.assertEqual(len(lines), 3) - self.assertEqual(lines[0], 'MD5 (afile) = ahash\n') - self.assertEqual(lines[1], 'MD5 (anotherfile) = anotherhash\n') - self.assertEqual(lines[2], 'MD5 (thirdfile) = thirdhash\n') - - def test_write_over(self): - lines = ['ahash afile\n', 'anotherhash anotherfile\n'] - - with open(self.sourcesfile, 'w') as f: - for line in lines: - f.write(line) - - s = sources.SourcesFile(self.sourcesfile, 'bsd', replace=True) - s.add_entry('md5', 'thirdfile', 'thirdhash') - s.write() - - with open(self.sourcesfile) as f: - lines = f.readlines() - - self.assertEqual(len(lines), 1) - self.assertEqual(lines[0], 'MD5 (thirdfile) = thirdhash\n') diff --git a/test/test_utils.py b/test/test_utils.py deleted file mode 100644 index 96f6bdf..0000000 --- a/test/test_utils.py +++ /dev/null @@ -1,202 +0,0 @@ -import unittest -import warnings - -import mock - -from pyrpkg.utils import cached_property, warn_deprecated, log_result - - -class CachedPropertyTestCase(unittest.TestCase): - def test_computed_only_once(self): - class Foo(object): - @cached_property - def foo(self): - runs.append("run once") - return 42 - - runs = [] - - f = Foo() - self.assertEqual(len(runs), 0) - self.assertEqual(f.foo, 42) - self.assertEqual(len(runs), 1) - self.assertEqual(f.foo, 42) - self.assertEqual(len(runs), 1) - - def test_not_shared_between_properties(self): - class Foo(object): - @cached_property - def foo(self): - foo_runs.append("run once") - return 42 - - @cached_property - def bar(self): - bar_runs.append("run once") - return 43 - - foo_runs = [] - bar_runs = [] - - f = Foo() - self.assertEqual(len(foo_runs), 0) - self.assertEqual(f.foo, 42) - self.assertEqual(len(foo_runs), 1) - self.assertEqual(f.foo, 42) - self.assertEqual(len(foo_runs), 1) - - self.assertEqual(len(bar_runs), 0) - self.assertEqual(f.bar, 43) - self.assertEqual(len(bar_runs), 1) - self.assertEqual(f.bar, 43) - self.assertEqual(len(bar_runs), 1) - - def test_not_shared_between_instances(self): - class Foo(object): - @cached_property - def foo(self): - foo_runs.append("run once") - return 42 - - class Bar(object): - @cached_property - def foo(self): - bar_runs.append("run once") - return 43 - - foo_runs = [] - bar_runs = [] - - f = Foo() - self.assertEqual(len(foo_runs), 0) - self.assertEqual(f.foo, 42) - self.assertEqual(len(foo_runs), 1) - self.assertEqual(f.foo, 42) - self.assertEqual(len(foo_runs), 1) - - b = Bar() - self.assertEqual(len(bar_runs), 0) - self.assertEqual(b.foo, 43) - self.assertEqual(len(bar_runs), 1) - self.assertEqual(b.foo, 43) - self.assertEqual(len(bar_runs), 1) - - def test_not_shared_when_inheriting(self): - class Foo(object): - @cached_property - def foo(self): - foo_runs.append("run once") - return 42 - - class Bar(Foo): - @cached_property - def foo(self): - bar_runs.append("run once") - return 43 - - foo_runs = [] - bar_runs = [] - - b = Bar() - self.assertEqual(len(bar_runs), 0) - self.assertEqual(b.foo, 43) - self.assertEqual(len(bar_runs), 1) - self.assertEqual(b.foo, 43) - self.assertEqual(len(bar_runs), 1) - - f = Foo() - self.assertEqual(len(foo_runs), 0) - self.assertEqual(f.foo, 42) - self.assertEqual(len(foo_runs), 1) - self.assertEqual(f.foo, 42) - self.assertEqual(len(foo_runs), 1) - - bar_runs = [] - b = Bar() - self.assertEqual(len(bar_runs), 0) - self.assertEqual(b.foo, 43) - self.assertEqual(len(bar_runs), 1) - self.assertEqual(b.foo, 43) - self.assertEqual(len(bar_runs), 1) - - -class DeprecationUtilsTestCase(unittest.TestCase): - def setUp(self): - warnings.simplefilter('always', DeprecationWarning) - - @mock.patch('sys.stderr') - def test_warn_deprecated(self, mock_stderr): - class Foo(object): - def old_method(self): - warn_deprecated(self.__class__.__name__, 'old_method', - 'new_method') - return self.new_method() - - def new_method(self): - return "Yay!" - - def mock_write(msg): - written_lines.append(msg) - - written_lines = [] - mock_stderr.write.side_effect = mock_write - - foo = Foo() - self.assertEqual(foo.old_method(), foo.new_method()) - self.assertEqual(len(written_lines), 1) - self.assertTrue('DeprecationWarning' in written_lines[0]) - self.assertTrue('Foo.old_method' in written_lines[0]) - self.assertTrue('Foo.new_method' in written_lines[0]) - - warnings.simplefilter('error', DeprecationWarning) - self.assertRaises(DeprecationWarning, foo.old_method) - self.assertEqual(len(written_lines), 1) - - -class LogResultTestCase(unittest.TestCase): - def setUp(self): - self.logs = [] - - def info(msg): - self.logs.append(msg) - - self.log_func = info - - def test_dict_result(self): - obj = {'spam': 'maps'} - expected = [ - 'spam:', - ' maps', - ] - log_result(self.log_func, obj) - self.assertEqual(self.logs, expected) - - def test_list_result(self): - obj = ['eggs', 'bacon', 'hash'] - expected = [ - 'eggs', - 'bacon', - 'hash', - ] - log_result(self.log_func, obj) - self.assertEqual(self.logs, expected) - - def test_str_result(self): - obj = 'spam' - expected = [ - 'spam', - ] - log_result(self.log_func, obj) - self.assertEqual(self.logs, expected) - - def test_complex_result(self): - obj = {'breakfast': ['eggs', 'bacon', {'spam': 'maps'}]} - expected = [ - 'breakfast:', - ' eggs', - ' bacon', - ' spam:', - ' maps', - ] - log_result(self.log_func, obj) - self.assertEqual(self.logs, expected) diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 0000000..380dc78 --- /dev/null +++ b/tests/commands/__init__.py @@ -0,0 +1,118 @@ +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + + +class CommandTestCase(unittest.TestCase): + def setUp(self): + self.origin_dir = os.getcwd() + self.path = tempfile.mkdtemp(prefix='rpkg-tests.') + self.gitroot = os.path.join(self.path, 'gitroot') + + self.module = 'module1' + + self.anongiturl = 'file://%s/%%(module)s' % self.gitroot + self.branchre = r'master|rpkg-tests-.+' + self.quiet = False + + # TODO: Figure out how to handle this + self.lookaside = 'TODO' + self.lookasidehash = 'md5' + self.lookaside_cgi = 'TODO' + self.gitbaseurl = 'TODO' + self.kojiconfig = 'TODO' + self.build_client = 'TODO' + self.clone_config = ''' + bz.default-component %(module)s + sendemail.to %(module)s-owner@fedoraproject.org + ''' + self.user = 'TODO' + self.dist = 'TODO' + self.target = 'TODO' + + def tearDown(self): + os.chdir(self.origin_dir) + shutil.rmtree(self.path) + + def make_new_git(self, module, branches=None): + """Make a new git repo, so that tests can clone it + + This is not a test method. + """ + if branches is None: + branches = [] + + # Create a bare Git repository + moduledir = os.path.join(self.gitroot, module) + os.makedirs(moduledir) + subprocess.check_call(['git', 'init', '--bare'], cwd=moduledir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Clone it, and do the minimal Dist Git setup + cloneroot = os.path.join(self.path, 'clonedir') + os.makedirs(cloneroot) + subprocess.check_call(['git', 'clone', 'file://%s' % moduledir], + cwd=cloneroot, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + clonedir = os.path.join(cloneroot, module.split('/')[-1]) + open(os.path.join(clonedir, '.gitignore'), 'w').close() + open(os.path.join(clonedir, 'sources'), 'w').close() + subprocess.check_call(['git', 'add', '.gitignore', 'sources'], + cwd=clonedir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.check_call(['git', 'commit', '-m', + 'Initial setup of the repo'], cwd=clonedir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.check_call(['git', 'push', 'origin', 'master'], + cwd=clonedir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # Add the requested branches + for branch in branches: + subprocess.check_call(['git', 'branch', branch], cwd=clonedir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.check_call(['git', 'push', 'origin', branch], + cwd=clonedir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # Drop the clone + shutil.rmtree(cloneroot) + + def get_tags(self, gitdir): + result = [] + + tags = subprocess.Popen(['git', 'tag', '-n1'], cwd=gitdir, + stdout=subprocess.PIPE, + universal_newlines=True).communicate()[0] + + for line in tags.split('\n'): + if not line: + continue + + tokens = [x for x in line.split() if x] + result.append([tokens[0], ' '.join(tokens[1:])]) + + return result + + def hijack_stdout(self): + class cm(object): + def __enter__(self): + from six.moves import cStringIO as StringIO + + self.old_stdout = sys.stdout + self.out = StringIO() + sys.stdout = self.out + + return self.out + + def __exit__(self, *args): + sys.stdout.flush() + sys.stdout = self.old_stdout + + self.out.seek(0) + + return cm() diff --git a/tests/commands/test_add_tag.py b/tests/commands/test_add_tag.py new file mode 100644 index 0000000..6e13404 --- /dev/null +++ b/tests/commands/test_add_tag.py @@ -0,0 +1,162 @@ +import os + +from . import CommandTestCase + + +class CommandAddTagTestCase(CommandTestCase): + def setUp(self): + super(CommandAddTagTestCase, self).setUp() + if 'GIT_EDITOR' in os.environ: + self.old_git_editor = os.environ['GIT_EDITOR'] + else: + self.old_git_editor = None + + def tearDown(self): + if self.old_git_editor is not None: + os.environ['GIT_EDITOR'] = self.old_git_editor + super(CommandAddTagTestCase, self).tearDown() + + def test_add_tag(self): + self.make_new_git(self.module) + + tag = 'v1.0' + message = 'This is a release' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + # `git tag` will call $EDITOR to ask the user to write a message + os.environ['GIT_EDITOR'] = ('/usr/bin/python -c "import sys; ' + 'open(sys.argv[1], \'w\').write(\'%s\')"' + % message) + + cmd.add_tag(tag) + + self.assertEqual(self.get_tags(moduledir), [[tag, message]]) + + def test_add_tag_with_message(self): + self.make_new_git(self.module) + + tag = 'v1.0' + message = 'This is a release' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + cmd.add_tag(tag, message=message) + + self.assertEqual(self.get_tags(moduledir), [[tag, message]]) + + def test_add_tag_with_message_from_file(self): + self.make_new_git(self.module) + + tag = 'v1.0' + message = 'This is a release' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + message_file = os.path.join(moduledir, 'tag_message') + + with open(message_file, 'w') as f: + f.write(message) + + cmd.add_tag(tag, file=message_file) + + self.assertEqual(self.get_tags(moduledir), [[tag, message]]) + + def test_add_tag_fails_with_existing(self): + self.make_new_git(self.module) + + tag = 'v1.0' + message = 'This is a release' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + cmd.add_tag(tag, message=message) + + # Now add the same tag again + def raises(): + cmd.add_tag(tag, message='No, THIS is a release') + + self.assertRaises(pyrpkg.rpkgError, raises) + + def test_add_tag_force_replace_existing(self): + self.make_new_git(self.module) + + tag = 'v1.0' + message = 'This is a release' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + cmd.add_tag(tag, message=message) + + # Now add the same tag again by force + newmessage = 'No, THIS is a release' + cmd.add_tag(tag, message=newmessage, force=True) + + self.assertEqual(self.get_tags(moduledir), [[tag, newmessage]]) + + def test_add_tag_many(self): + self.make_new_git(self.module) + + tags = [['v1.0', 'This is a release'], + ['v2.0', 'This is another release']] + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + for tag, message in tags: + cmd.add_tag(tag, message=message) + + self.assertEqual(self.get_tags(moduledir), tags) diff --git a/tests/commands/test_check_repo.py b/tests/commands/test_check_repo.py new file mode 100644 index 0000000..0ca24df --- /dev/null +++ b/tests/commands/test_check_repo.py @@ -0,0 +1,73 @@ +import os +import shutil +import subprocess +import tempfile + +from pyrpkg.errors import rpkgError + +from . import CommandTestCase + + +class CheckRepoCase(CommandTestCase): + + def setUp(self): + super(CheckRepoCase, self).setUp() + self.dist = "master" + self.make_new_git(self.module) + moduledir = os.path.join(self.gitroot, self.module) + + self.altpath = tempfile.mkdtemp(prefix='rpkg-tests.') + self.clonedir = os.path.join(self.altpath, self.module) + subprocess.check_call(['git', 'clone', 'file://%s' % moduledir], + cwd=self.altpath, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + import pyrpkg + self.cmd = pyrpkg.Commands( + self.clonedir, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet + ) + + def tearDown(self): + super(CheckRepoCase, self).tearDown() + # Drop the clone + shutil.rmtree(self.altpath) + + def test_repo_is_dirty(self): + with open(os.path.join(self.clonedir, 'sources'), 'w') as fd: + fd.write("a") + + try: + self.cmd.check_repo(is_dirty=True, all_pushed=False) + except rpkgError as exception: + self.assertTrue("has uncommitted changes" in str(exception)) + else: + self.fail("Expected an rpkgError exception.") + + def test_repo_has_unpushed_changes(self): + with open(os.path.join(self.clonedir, 'sources'), 'w') as fd: + fd.write("a") + subprocess.check_call( + ['git', 'add', 'sources'], + cwd=self.clonedir + ) + subprocess.check_call( + ['git', 'commit', '-m', 'commit sources'], + cwd=self.clonedir, + ) + + try: + self.cmd.check_repo(is_dirty=False, all_pushed=True) + except rpkgError as exception: + self.assertTrue("There are unpushed changes in your repo" in + str(exception)) + else: + self.fail("Expected an rpkgError exception.") + + def test_repo_is_clean(self): + self.cmd.check_repo(is_dirty=True, all_pushed=False) + + def test_repo_has_everything_pushed(self): + self.cmd.check_repo(is_dirty=False, all_pushed=True) diff --git a/tests/commands/test_clone.py b/tests/commands/test_clone.py new file mode 100644 index 0000000..5541a9b --- /dev/null +++ b/tests/commands/test_clone.py @@ -0,0 +1,157 @@ +import os +import shutil +import tempfile + +import git + +from . import CommandTestCase + + +CLONE_CONFIG = ''' + bz.default-component %(module)s + sendemail.to %(module)s-owner@fedoraproject.org +''' + + +class CommandCloneTestCase(CommandTestCase): + def test_clone_anonymous(self): + self.make_new_git(self.module) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone_config = CLONE_CONFIG + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git'))) + confgit = git.Git(moduledir) + self.assertEqual(confgit.config('bz.default-component'), self.module) + self.assertEqual(confgit.config('sendemail.to'), + "%s-owner@fedoraproject.org" % self.module) + + def test_clone_anonymous_with_namespace(self): + self.module = 'rpms/module1' + self.make_new_git(self.module) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet, distgit_namespaced=True) + cmd.clone_config = CLONE_CONFIG + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, 'module1') + self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git'))) + confgit = git.Git(moduledir) + self.assertEqual(confgit.config('bz.default-component'), self.module) + self.assertEqual(confgit.config('sendemail.to'), + "%s-owner@fedoraproject.org" % self.module) + + def test_clone_anonymous_with_path(self): + self.make_new_git(self.module) + + altpath = tempfile.mkdtemp(prefix='rpkg-tests.') + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True, path=altpath) + + moduledir = os.path.join(altpath, self.module) + self.assertTrue(os.path.isdir(os.path.join(moduledir, '.git'))) + + notmoduledir = os.path.join(self.path, self.module) + self.assertFalse(os.path.isdir(os.path.join(notmoduledir, '.git'))) + + shutil.rmtree(altpath) + + def test_clone_anonymous_with_branch(self): + self.make_new_git(self.module, + branches=['rpkg-tests-1', 'rpkg-tests-2']) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True, branch='rpkg-tests-1') + + with open(os.path.join( + self.path, self.module, '.git', 'HEAD')) as HEAD: + self.assertEqual(HEAD.read(), 'ref: refs/heads/rpkg-tests-1\n') + + def test_clone_anonymous_with_bare_dir(self): + self.make_new_git(self.module) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True, bare_dir='%s.git' % self.module) + + clonedir = os.path.join(self.path, '%s.git' % self.module) + self.assertTrue(os.path.isdir(clonedir)) + self.assertFalse(os.path.exists(os.path.join(clonedir, 'index'))) + + def test_clone_fails_with_both_branch_and_bare_dir(self): + self.make_new_git(self.module, + branches=['rpkg-tests-1', 'rpkg-tests-2']) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + + def raises(): + cmd.clone(self.module, anon=True, branch='rpkg-tests-1', + bare_dir='test.git') + self.assertRaises(pyrpkg.rpkgError, raises) + + def test_clone_into_dir(self): + self.make_new_git(self.module, + branches=['rpkg-tests-1', 'rpkg-tests-2']) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone( + self.module, anon=True, branch='rpkg-tests-1', target='new_clone') + + with open(os.path.join( + self.path, 'new_clone', '.git', 'HEAD')) as HEAD: + self.assertEqual(HEAD.read(), 'ref: refs/heads/rpkg-tests-1\n') + + def test_clone_into_dir_with_namespace(self): + self.module = 'rpms/module1' + self.make_new_git(self.module, + branches=['rpkg-tests-1', 'rpkg-tests-2']) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet, distgit_namespaced=True) + cmd.clone( + self.module, anon=True, branch='rpkg-tests-1', target='new_clone') + + with open(os.path.join( + self.path, 'new_clone', '.git', 'HEAD')) as HEAD: + self.assertEqual(HEAD.read(), 'ref: refs/heads/rpkg-tests-1\n') diff --git a/tests/commands/test_delete_tag.py b/tests/commands/test_delete_tag.py new file mode 100644 index 0000000..2141059 --- /dev/null +++ b/tests/commands/test_delete_tag.py @@ -0,0 +1,52 @@ +import os + +from . import CommandTestCase + + +class CommandDeleteTagTestCase(CommandTestCase): + def test_delete_tag(self): + self.make_new_git(self.module) + + tag = 'v1.0' + message = 'This is a release' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + # First, add a tag + cmd.add_tag(tag, message=message) + self.assertEqual(self.get_tags(moduledir), [[tag, message]]) + + # Now delete it + cmd.delete_tag(tag) + tags = [t for (t, m) in self.get_tags(moduledir)] + self.assertFalse(tag in tags) + + def test_delete_tag_fails_inexistent(self): + self.make_new_git(self.module) + + tag = 'v1.0' + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + # Try deleting an inexistent tag + def raises(): + cmd.delete_tag(tag) + self.assertRaises(pyrpkg.rpkgError, raises) diff --git a/tests/commands/test_list_tag.py b/tests/commands/test_list_tag.py new file mode 100644 index 0000000..8135f5e --- /dev/null +++ b/tests/commands/test_list_tag.py @@ -0,0 +1,159 @@ +import os + +from . import CommandTestCase + + +class CommandListTagTestCase(CommandTestCase): + def test_list_tag_no_tags(self): + self.make_new_git(self.module) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + with self.hijack_stdout() as out: + cmd.list_tag() + + self.assertEqual(out.read().strip(), '') + + def test_list_tag_many(self): + self.make_new_git(self.module) + + tags = [['v1.0', 'This is a release'], + ['v2.0', 'This is another release']] + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + for tag, message in tags: + cmd.add_tag(tag, message=message) + + with self.hijack_stdout() as out: + cmd.list_tag() + + result = out.read().strip().split('\n') + + self.assertEqual(result, [t for (t, m) in tags]) + + def test_list_tag_specific(self): + self.make_new_git(self.module) + + tags = [['v1.0', 'This is a release'], + ['v2.0', 'This is another release']] + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + for tag, message in tags: + cmd.add_tag(tag, message=message) + + with self.hijack_stdout() as out: + cmd.list_tag(tagname='v1.0') + + result = out.read().strip().split('\n') + + self.assertEqual(result, ['v1.0']) + + def test_list_tag_inexistent(self): + self.make_new_git(self.module) + + tags = [['v1.0', 'This is a release'], + ['v2.0', 'This is another release']] + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + for tag, message in tags: + cmd.add_tag(tag, message=message) + + with self.hijack_stdout() as out: + cmd.list_tag(tagname='v1.1') + + result = out.read().strip().split('\n') + + self.assertEqual(result, ['']) + + def test_list_tag_glob(self): + self.make_new_git(self.module) + + tags = [['v1.0', 'This is a release'], + ['v2.0', 'This is another release']] + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + for tag, message in tags: + cmd.add_tag(tag, message=message) + + with self.hijack_stdout() as out: + cmd.list_tag(tagname='v1*') + + result = out.read().strip().split('\n') + + self.assertEqual(result, ['v1.0']) + + def test_list_tag_wildcard(self): + self.make_new_git(self.module) + + tags = [['v1.0', 'This is a release'], + ['v2.0', 'This is another release']] + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + for tag, message in tags: + cmd.add_tag(tag, message=message) + + with self.hijack_stdout() as out: + cmd.list_tag(tagname='*') + + result = out.read().strip().split('\n') + + self.assertEqual(result, [t for (t, m) in tags]) diff --git a/tests/commands/test_package_name.py b/tests/commands/test_package_name.py new file mode 100644 index 0000000..23cdb69 --- /dev/null +++ b/tests/commands/test_package_name.py @@ -0,0 +1,26 @@ +import os + +import six + +from . import CommandTestCase + + +class CommandPackageNameTestCase(CommandTestCase): + def test_name_is_not_unicode(self): + self.make_new_git(self.module) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone(self.module, anon=True) + + moduledir = os.path.join(self.path, self.module) + cmd.path = moduledir + + # pycurl can't handle unicode variable + # module_name needs to be a byte string + self.assertNotEqual(type(cmd.module_name), six.text_type) + self.assertEqual(type(cmd.module_name), six.binary_type) diff --git a/tests/commands/test_patch.py b/tests/commands/test_patch.py new file mode 100644 index 0000000..878871a --- /dev/null +++ b/tests/commands/test_patch.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +import six + +from . import CommandTestCase + + +class CommandPatchTestCase(CommandTestCase): + def setUp(self): + super(CommandPatchTestCase, self).setUp() + self.text_ascii = "Lorem ipsum dolor sit amet, consectetur elit.\n" \ + "Sed vel enim nec tortor posuere sodales sit amet mauris.\n" \ + "Duis ipsum dui, consectetur pretium a, vestibulum.\n"\ + "Nunc vel consectetur libero. Aenean , metus quis posuere\n" \ + "vulputate, purus metus fringilla, sit amet interdum tellus\n" + self.text_utf8 = "ěšč\n" \ + "ščř\n" \ + "ýáí" + if six.PY3: + self.text_utf8 = self.text_utf8.encode("utf-8") + + def test_byte_offset_first_line(self): + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + line, offset = cmd._byte_offset_to_line_number(self.text_ascii, 10) + # 10 byte offset mean line 1 and character 11 + self.assertEqual(line, 1) + self.assertEqual(offset, 11) + + def test_byte_offset_next_line(self): + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + + line, offset = cmd._byte_offset_to_line_number(self.text_ascii, 46) + # 46 byte offset is first character on second line + self.assertEqual(line, 2) + self.assertEqual(offset, 1) + + def test_byte_offset_utf8(self): + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + text = self.text_utf8.decode('UTF-8', 'ignore') + line, offset = cmd._byte_offset_to_line_number(text, 9) + # 9 byte offset mean line 3 and second character + self.assertEqual(line, 3) + self.assertEqual(offset, 2) diff --git a/tests/commands/test_push.py b/tests/commands/test_push.py new file mode 100644 index 0000000..0c681f7 --- /dev/null +++ b/tests/commands/test_push.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- + +import os +import git + +from . import CommandTestCase + + +SPECFILE_TEMPLATE = """Name: test +Version: 1.0 +Release: 1.0 +Summary: test + +Group: Applications/System +License: GPLv2+ + +%s + +%%description +Test + +%%install +rm -f $RPM_BUILD_ROOT%%{_sysconfdir}/""" + +CLONE_CONFIG = ''' + bz.default-component %(module)s + sendemail.to %(module)s-owner@fedoraproject.org +''' + + +class CommandPushTestCase(CommandTestCase): + + def setUp(self): + # Tests within this case would change working directory. Changing back + # to original directory to avoid any potential problems. + self.original_dir = os.path.abspath(os.curdir) + super(CommandPushTestCase, self).setUp() + + def tearDown(self): + os.chdir(self.original_dir) + super(CommandPushTestCase, self).tearDown() + + def test_push_outside_repo(self): + """push from outside repo with --path option""" + + self.make_new_git(self.module) + + import pyrpkg + cmd = pyrpkg.Commands(self.path, self.lookaside, + self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + cmd.clone_config = CLONE_CONFIG + cmd.clone(self.module, anon=True) + cmd.path = os.path.join(self.path, self.module) + os.chdir(os.path.join(self.path, self.module)) + + spec_file = 'module.spec' + with open(spec_file, 'w') as f: + f.write(SPECFILE_TEMPLATE % '') + + cmd.repo.index.add([spec_file]) + cmd.repo.index.commit("add SPEC") + + # Now, change directory to parent and test the push + os.chdir(self.path) + cmd.push() + + +class TestPushWithPatches(CommandTestCase): + + def setUp(self): + super(TestPushWithPatches, self).setUp() + + self.make_new_git(self.module) + + import pyrpkg + self.cmd = pyrpkg.Commands(self.path, self.lookaside, + self.lookasidehash, + self.lookaside_cgi, self.gitbaseurl, + self.anongiturl, self.branchre, + self.kojiconfig, + self.build_client, self.user, self.dist, + self.target, self.quiet) + self.cmd.clone_config = CLONE_CONFIG + self.cmd.clone(self.module, anon=True) + self.cmd.path = os.path.join(self.path, self.module) + os.chdir(os.path.join(self.path, self.module)) + + # Track SPEC and a.patch in git + spec_file = 'module.spec' + with open(spec_file, 'w') as f: + f.write(SPECFILE_TEMPLATE % '''Patch0: a.patch +Patch1: b.path +Patch2: c.path +Patch3: d.path +''') + + for patch_file in ('a.patch', 'b.patch', 'c.patch', 'd.patch'): + with open(patch_file, 'w') as f: + f.write(patch_file) + + # Track c.patch in sources + from pyrpkg.sources import SourcesFile + sources_file = SourcesFile(self.cmd.sources_filename, + self.cmd.source_entry_type) + file_hash = self.cmd.lookasidecache.hash_file('c.patch') + sources_file.add_entry(self.cmd.lookasidehash, 'c.patch', file_hash) + sources_file.write() + + self.cmd.repo.index.add([spec_file, 'a.patch', 'sources']) + self.cmd.repo.index.commit('add SPEC and patches') + + def test_find_untracked_patches(self): + untracked_patches = self.cmd.find_untracked_patches() + untracked_patches.sort() + self.assertEqual(['b.patch', 'd.patch'], untracked_patches) + + def test_push_not_blocked_by_untracked_patches(self): + self.cmd.push() + + # Verify added files are pushed to origin + origin_repo_path = self.cmd.repo.git.config( + '--get', 'remote.origin.url').replace('file://', '') + origin_repo = git.Repo(origin_repo_path) + git_tree = origin_repo.head.commit.tree + self.assertTrue('a.patch' in git_tree) + self.assertTrue('b.patch' not in git_tree) + self.assertTrue('c.patch' not in git_tree) + self.assertTrue('d.patch' not in git_tree) + + sources_content = origin_repo.git.show('master:sources').strip() + with open('sources', 'r') as f: + expected_sources_content = f.read().strip() + self.assertEqual(expected_sources_content, sources_content) diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..7267c89 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,432 @@ +# -*- coding: utf-8 -*- + +import os +import shutil +import tempfile +import unittest +import subprocess + +import git +from mock import patch + +from pyrpkg import Commands +from pyrpkg import rpkgError + +# Following global variables are used to construct Commands for tests in this +# module. Only for testing purpose, and they are not going to be used for +# hitting real services. +lookaside = 'http://dist-git-qa.server/repo/pkgs' +lookaside_cgi = 'http://dist-git-qa.server/lookaside/upload.cgi' +gitbaseurl = 'ssh://%(user)s@dist-git-qa.server/rpms/%(module)s' +anongiturl = 'git://dist-git-qa.server/rpms/%(module)s' +lookasidehash = 'md5' +branchre = 'rhel' +kojiconfig = '/etc/koji.conf.d/brewstage.conf' +build_client = 'brew-stage' + +spec_file = ''' +Summary: Dummy summary +Name: docpkg +Version: 1.2 +Release: 2 +License: GPL +Group: Applications/Productivity +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +%description +This is a dummy description. +%prep +%build +%clean +rm -rf $$RPM_BUILD_ROOT +%install +rm -rf $RPM_BUILD_ROOT +mkdir $RPM_BUILD_ROOT +%files +%changelog +* Thu Apr 21 2006 Chenxiong Qi - 1.2-2 +- Initial version +''' + + +def run(cmd, **kwargs): + returncode = subprocess.call(cmd, **kwargs) + if returncode != 0: + raise RuntimeError('Command fails. Command: %s. Return code %d' % ( + ' '.join(cmd), returncode)) + + +class CommandTestCase(unittest.TestCase): + + def setUp(self): + # create a base repo + self.repo_path = tempfile.mkdtemp(prefix='rpkg-commands-tests-') + + # Add spec file to this repo and commit + spec_file_path = os.path.join(self.repo_path, 'package.spec') + with open(spec_file_path, 'w') as f: + f.write(spec_file) + + git_cmds = [ + ['git', 'init'], + ['git', 'add', spec_file_path], + ['git', 'config', 'user.email', 'cqi@redhat.com'], + ['git', 'config', 'user.name', 'Chenxiong Qi'], + ['git', 'commit', '-m', '"initial commit"'], + ['git', 'branch', 'eng-rhel-6'], + ['git', 'branch', 'eng-rhel-6.5'], + ['git', 'branch', 'eng-rhel-7'], + ] + for cmd in git_cmds: + run(cmd, cwd=self.repo_path) + + # Clone the repo + self.cloned_repo_path = tempfile.mkdtemp(prefix='rpkg-commands-tests-cloned-') + git_cmds = [ + ['git', 'clone', self.repo_path, self.cloned_repo_path], + ['git', 'branch', '--track', 'eng-rhel-6', 'origin/eng-rhel-6'], + ['git', 'branch', '--track', 'eng-rhel-6.5', 'origin/eng-rhel-6.5'], + ['git', 'branch', '--track', 'eng-rhel-7', 'origin/eng-rhel-7'], + ] + for cmd in git_cmds: + run(cmd, cwd=self.cloned_repo_path) + + def tearDown(self): + shutil.rmtree(self.repo_path) + shutil.rmtree(self.cloned_repo_path) + + def make_commands(self, path=None, user=None, dist=None, target=None, quiet=None): + """Helper method for creating Commands object for test cases + + This is where you should extend to add more features to support + additional requirements from other Commands specific test cases. + + Some tests need customize one of user, dist, target, and quiet options + when creating an instance of Commands. Keyword arguments user, dist, + target, and quiet here is for this purpose. + + :param str path: path to repository where this Commands will work on + top of + :param str user: user passed to --user option + :param str dist: dist passed to --dist option + :param str target: target passed to --target option + :param str quiet: quiet passed to --quiet option + """ + _repo_path = path if path else self.cloned_repo_path + return Commands(_repo_path, + lookaside, lookasidehash, lookaside_cgi, + gitbaseurl, anongiturl, + branchre, + kojiconfig, build_client, + user=user, dist=dist, target=target, quiet=quiet) + + def checkout_branch(self, repo, branch_name): + """Checkout to a local branch + + :param git.Repo repo: `git.Repo` instance represents a git repository + that current code works on top of. + :param str branch_name: name of local branch to checkout + """ + heads = [head for head in repo.heads if head.name == branch_name] + assert len(heads) > 0, \ + 'Repo must have a local branch named {} that ' \ + 'is for running tests. But now, it does not exist. Please check ' \ + 'if the repo is correct.'.format(branch_name) + + heads[0].checkout() + + def create_branch(self, repo, branch_name): + repo.git.branch(branch_name) + + def make_a_dummy_commit(self, repo): + filename = os.path.join(repo.working_dir, 'document.txt') + with open(filename, 'a+') as f: + f.write('Hello rpkg') + repo.index.add([filename]) + repo.index.commit('update document') + + +def mock_load_rpmdefines(self): + """Mock Commands.load_rpmdefines by setting empty list to _rpmdefines + + :param Commands self: load_rpmdefines is an instance method of Commands, + self is the instance whish is calling this method. + """ + self._rpmdefines = [] + + +def mock_load_spec(fake_spec): + """Return a mocked load_spec method that sets a fake spec to Commands + + :param str fake_spec: an arbitrary string representing a fake spec + file. What value is passed to fake_spec depends on the test purpose + completely. + """ + def mocked_load_spec(self): + """Mocked load_spec to set fake spec to an instance of Commands + + :param Commands self: load_spec is an instance method of Commands, self + is the instance which is calling this method. + """ + self._spec = fake_spec + return mocked_load_spec + + +def mock_load_branch_merge(fake_branch_merge): + """Return a mocked load_branch_merge method + + The mocked method sets a fake branch name to _branch_merge. + + :param str fake_branch_merge: an arbitrary string representing a fake + branch name. What value should be passed to fake_branch_merge depends on + the test purpose completely. + """ + def mocked_method(self): + """ + Mocked load_branch_merge to set fake branch name to an instance of + Commands. + + :param Commands self: load_branch_merge is an instance method of + Commands, so self is the instance which is calling this method. + """ + self._branch_merge = fake_branch_merge + return mocked_method + + +class LoadNameVerRelTest(CommandTestCase): + """Test case for Commands.load_nameverrel""" + + def setUp(self): + super(LoadNameVerRelTest, self).setUp() + self.cmd = self.make_commands() + self.checkout_branch(self.cmd.repo, 'eng-rhel-6') + + def test_load_from_spec(self): + """Ensure name, version, release can be loaded from a valid SPEC""" + self.cmd.load_nameverrel() + self.assertEqual('docpkg', self.cmd._module_name_spec) + self.assertEqual('0', self.cmd._epoch) + self.assertEqual('1.2', self.cmd._ver) + self.assertEqual('2', self.cmd._rel) + + def test_load_spec_where_path_contains_space(self): + """Ensure load_nameverrel works with a repo whose path contains space + + This test aims to test the space appearing in path does not break rpm + command execution. + + For this test purpose, firstly, original repo has to be cloned to a + new place which has a name containing arbitrary spaces. + """ + cloned_repo_dir = '/tmp/rpkg test cloned repo' + if os.path.exists(cloned_repo_dir): + shutil.rmtree(cloned_repo_dir) + cloned_repo = self.cmd.repo.clone(cloned_repo_dir) + + # Switching to branch eng-rhel-6 explicitly is required by running this + # on RHEL6/7 because an old version of git is available in the + # repo. + # The failure reason is, old version of git makes the master as the + # active branch in cloned repository, whatever the current active + # branch is in the remote repository. + # As of fixing this, I ran test on Fedora 23 with git 2.5.5, and test + # fails on RHEL7 with git 1.8.3.1 + cloned_repo.git.checkout('eng-rhel-6') + + cmd = self.make_commands(path=cloned_repo_dir) + + cmd.load_nameverrel() + self.assertEqual('docpkg', cmd._module_name_spec) + self.assertEqual('0', cmd._epoch) + self.assertEqual('1.2', cmd._ver) + self.assertEqual('2', cmd._rel) + + @patch('pyrpkg.Commands.load_rpmdefines', new=mock_load_rpmdefines) + @patch('pyrpkg.Commands.load_spec', + new=mock_load_spec('unknown-rpm-option a-nonexistent-package.spec')) + def test_load_when_rpm_fails(self): + """Ensure rpkgError is raised when rpm command fails + + Commands.load_spec is mocked to help generate an incorrect rpm command + line to cause the error that this test expects. + + Test test does not care about what rpm defines are retrieved from + repository, so setting an empty list to Commands._rpmdefines is safe + and enough. + """ + self.assertRaises(rpkgError, self.cmd.load_nameverrel) + + +class LoadBranchMergeTest(CommandTestCase): + """Test case for testing Commands.load_branch_merge""" + + def setUp(self): + super(LoadBranchMergeTest, self).setUp() + + self.cmd = self.make_commands() + + def test_load_branch_merge_from_eng_rhel_6(self): + self.checkout_branch(self.cmd.repo, 'eng-rhel-6') + self.cmd.load_branch_merge() + self.assertEqual(self.cmd._branch_merge, 'eng-rhel-6') + + def test_load_branch_merge_from_eng_rhel_6_5(self): + """ + Ensure load_branch_merge can work well against a more special branch + eng-rhel-6.5 + """ + self.checkout_branch(self.cmd.repo, 'eng-rhel-6.5') + self.cmd.load_branch_merge() + self.assertEqual(self.cmd._branch_merge, 'eng-rhel-6.5') + + def test_load_branch_merge_from_not_remote_merge_branch(self): + """Ensure load_branch_merge fails against local-branch + + A new local branch named local-branch is created for this test, loading + branch merge from this local branch should fail because there is no + configuration item branch.local-branch.merge. + """ + self.create_branch(self.cmd.repo, 'local-branch') + self.checkout_branch(self.cmd.repo, 'local-branch') + try: + self.cmd.load_branch_merge() + except rpkgError as e: + self.assertEqual('Unable to find remote branch. Use --dist', str(e)) + else: + self.fail("It's expected to raise rpkgError, but not.") + + def test_load_branch_merge_using_dist_option(self): + """Ensure load_branch_merge uses dist specified via --dist + + Switch to eng-rhel-6 branch, that is valid for load_branch_merge and to + see if load_branch_merge still uses dist rather than such a valid + branch. + """ + self.checkout_branch(self.cmd.repo, 'eng-rhel-6') + + cmd = self.make_commands(dist='branch_merge') + cmd.load_branch_merge() + self.assertEqual('branch_merge', cmd._branch_merge) + + +class LoadRPMDefinesTest(CommandTestCase): + """Test case for Commands.load_rpmdefines""" + + def setUp(self): + super(LoadRPMDefinesTest, self).setUp() + self.cmd = self.make_commands() + + def assert_loaded_rpmdefines(self, branch_name, expected_defines): + self.checkout_branch(self.cmd.repo, branch_name) + + self.cmd.load_rpmdefines() + self.assertTrue(self.cmd._rpmdefines) + + # Convert defines into dict for assertion conveniently. The dict + # contains mapping from variable name to value. For example, + # { + # '_sourcedir': '/path/to/src-dir', + # '_specdir': '/path/to/spec', + # '_builddir': '/path/to/build-dir', + # '_srcrpmdir': '/path/to/srcrpm-dir', + # 'dist': 'el7' + # } + defines = dict([item.split(' ') for item in ( + define.replace("'", '').split(' ', 1)[1] for + define in self.cmd._rpmdefines)]) + + for var, val in expected_defines.items(): + self.assertTrue(var in defines) + self.assertEqual(val, defines[var]) + + def test_load_rpmdefines_from_eng_rhel_6(self): + """Run load_rpmdefines against branch eng-rhel-6""" + expected_rpmdefines = { + '_sourcedir': self.cloned_repo_path, + '_specdir': self.cloned_repo_path, + '_builddir': self.cloned_repo_path, + '_srcrpmdir': self.cloned_repo_path, + '_rpmdir': self.cloned_repo_path, + 'dist': u'.el6', + 'rhel': u'6', + 'el6': u'1', + } + self.assert_loaded_rpmdefines('eng-rhel-6', expected_rpmdefines) + + def test_load_rpmdefines_from_eng_rhel_6_5(self): + """Run load_rpmdefines against branch eng-rhel-6.5 + + Working on a different branch name is the only difference from test + method test_load_rpmdefines_from_eng_rhel_6. + """ + expected_rpmdefines = { + '_sourcedir': self.cloned_repo_path, + '_specdir': self.cloned_repo_path, + '_builddir': self.cloned_repo_path, + '_srcrpmdir': self.cloned_repo_path, + '_rpmdir': self.cloned_repo_path, + 'dist': u'.el6_5', + 'rhel': u'6', + 'el6_5': u'1', + } + self.assert_loaded_rpmdefines('eng-rhel-6.5', expected_rpmdefines) + + @patch('pyrpkg.Commands.load_branch_merge', + new=mock_load_branch_merge('invalid-branch-name')) + def test_load_rpmdefines_against_invalid_branch(self): + """Ensure load_rpmdefines if active branch name is invalid + + This test requires an invalid branch name even if + Commands.load_branch_merge is able to get it from current active + branch. So, I only care about the value returned from method + load_branch_merge, and just mock it and let it return the value this + test requires. + """ + self.assertRaises(rpkgError, self.cmd.load_rpmdefines) + + +class CheckRepoWithOrWithoutDistOptionCase(CommandTestCase): + """Check whether there are unpushed changes with or without specified dist + + Ensure check_repo works in a correct way to check if there are unpushed + changes, and this should not be affected by specified dist or not. + Bug 1169663 describes a concrete use case and this test case is designed + as what that bug describs. + """ + + def setUp(self): + super(CheckRepoWithOrWithoutDistOptionCase, self).setUp() + + private_branch = 'private-dev-branch' + origin_repo = git.Repo(self.repo_path) + origin_repo.git.checkout('master') + origin_repo.git.branch(private_branch) + self.make_a_dummy_commit(origin_repo) + + cloned_repo = git.Repo(self.cloned_repo_path) + cloned_repo.git.pull() + cloned_repo.git.checkout('-b', private_branch, 'origin/%s' % private_branch) + for i in xrange(3): + self.make_a_dummy_commit(cloned_repo) + cloned_repo.git.push() + + def test_check_repo_with_specificed_dist(self): + cmd = self.make_commands(self.cloned_repo_path, dist='eng-rhel-6') + try: + cmd.check_repo() + except rpkgError as e: + if 'There are unpushed changes in your repo' in e: + self.fail('There are unpushed changes in your repo. This ' + 'should not happen. Something must be going wrong.') + + self.fail('Should not fail. Something must be going wrong.') + + def test_check_repo_without_specificed_dist(self): + cmd = self.make_commands(self.cloned_repo_path) + try: + cmd.check_repo() + except rpkgError as e: + if 'There are unpushed changes in your repo' in e: + self.fail('There are unpushed changes in your repo. This ' + 'should not happen. Something must be going wrong.') + + self.fail('Should not fail. Something must be going wrong.') diff --git a/tests/test_gitgnore.py b/tests/test_gitgnore.py new file mode 100644 index 0000000..2fc7c17 --- /dev/null +++ b/tests/test_gitgnore.py @@ -0,0 +1,109 @@ +import os +import shutil +import tempfile +import unittest + + +class GitIgnoreTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') + + def tearDown(self): + shutil.rmtree(self.workdir) + + def test_add_existing_line(self): + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) + gi.add('a new line') + self.assertTrue(gi.modified) + + # Cheat a bit for unit tests + gi.modified = False + + gi.add('a new line') + self.assertFalse(gi.modified) + + gi.add('*') + self.assertTrue(gi.modified) + + # Cheat a bit for unit tests + gi.modified = False + + gi.add('something different') + self.assertFalse(gi.modified) + + def test_match_empty(self): + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) + self.assertFalse(gi.match('this does not exist')) + + # The empty string could match an empty file, but we don't want it to + self.assertFalse(gi.match('')) + + def test_match_line_from_existing_file(self): + gi_path = os.path.join(self.workdir, 'gitignore') + + with open(gi_path, 'w') as f: + f.write('this line exists\n') + + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(gi_path) + self.assertTrue(gi.match('this line exists')) + self.assertTrue(gi.match('this line exists\n')) + self.assertTrue(gi.match('/this line exists')) + self.assertTrue(gi.match('/this line exists\n')) + + self.assertFalse(gi.match('but this line does not')) + + def test_match_unwritten_line(self): + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) + gi.add('here is a new line') + + self.assertTrue(gi.modified) + self.assertTrue(gi.match('here is a new line')) + + def test_match_glob(self): + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(os.path.join(self.workdir, 'gitignore')) + gi.add('*') + + self.assertTrue(gi.match('Surely this is matched by a wildcard?')) + + def test_write_new_file(self): + gi_path = os.path.join(self.workdir, 'gitignore') + + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(gi_path) + gi.add('here is a new line') + gi.write() + + self.assertFalse(gi.modified) + + with open(gi_path) as f: + self.assertEqual(f.read(), 'here is a new line\n') + + def test_write_append_to_existing_file(self): + gi_path = os.path.join(self.workdir, 'gitignore') + + lines = ('this line exists', 'here is a new line') + + with open(gi_path, 'w') as f: + f.write(lines[0]) + + from pyrpkg.gitignore import GitIgnore + + gi = GitIgnore(gi_path) + gi.add(lines[1]) + gi.write() + + self.assertFalse(gi.modified) + + with open(gi_path) as f: + self.assertEqual(f.read(), '%s\n' % '\n'.join(lines)) diff --git a/tests/test_lookaside.py b/tests/test_lookaside.py new file mode 100644 index 0000000..7509e61 --- /dev/null +++ b/tests/test_lookaside.py @@ -0,0 +1,586 @@ +# Copyright (c) 2015 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +import hashlib +import os +import shutil +import tempfile +import unittest + +import mock +import pycurl + +from pyrpkg.lookaside import CGILookasideCache +from pyrpkg.errors import DownloadError, InvalidHashType, UploadError + + +class CGILookasideCacheTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') + self.filename = os.path.join(self.workdir, self._testMethodName) + + def tearDown(self): + shutil.rmtree(self.workdir) + + def test_hash_file(self): + lc = CGILookasideCache('sha512', '_', '_') + + with open(self.filename, 'w') as f: + f.write('something') + + result = lc.hash_file(self.filename, 'md5') + self.assertEqual(result, '437b930db84b8079c2dd804a71936b5f') + + result = lc.hash_file(self.filename) + self.assertEqual(result, '983d43ddff6da90f6a5d3b6172446a1ffe228b803fe64fdd5dcfab5646078a896851fe82f623c9d6e5654b3d2f363a04ec17cfb62b607437a9c7c132d511e522') # nopep8 + + def test_hash_file_invalid_hash_type(self): + lc = CGILookasideCache('sha512', '_', '_') + self.assertRaises(InvalidHashType, lc.hash_file, '_', 'sha42') + + def test_hash_file_empty(self): + lc = CGILookasideCache('sha512', '_', '_') + + with open(self.filename, 'w') as f: + f.write('') + + result = lc.hash_file(self.filename, 'md5') + self.assertEqual(result, 'd41d8cd98f00b204e9800998ecf8427e') + + def test_file_is_valid(self): + lc = CGILookasideCache('md5', '_', '_') + + with open(self.filename, 'w') as f: + f.write('something') + + self.assertTrue(lc.file_is_valid(self.filename, + '437b930db84b8079c2dd804a71936b5f')) + self.assertFalse(lc.file_is_valid(self.filename, 'not the right hash', + hashtype='sha512')) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_download(self, mock_curl): + def mock_getinfo(info): + return 200 if info == pycurl.RESPONSE_CODE else 0 + + def mock_perform(): + with open(self.filename, 'rb') as f: + curlopts[pycurl.WRITEDATA].write(f.read()) + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.side_effect = mock_getinfo + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + with open(self.filename, 'wb') as f: + f.write(b'content') + + name = 'pyrpkg' + filename = 'pyrpkg-0.0.tar.xz' + hash = hashlib.sha512(b'content').hexdigest() + outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') + full_url = 'http://example.com/%s/%s/%s/%s' % (name, filename, hash, + filename) + + lc = CGILookasideCache('sha512', 'http://example.com', '_') + lc.download(name, filename, hash, outfile, hashtype='sha512') + self.assertEqual(curl.perform.call_count, 1) + self.assertEqual(curlopts[pycurl.URL], full_url) + self.assertEqual(os.path.getmtime(outfile), 0) + + with open(outfile) as f: + self.assertEqual(f.read(), 'content') + + # Try a second time + lc.download(name, filename, hash, outfile) + self.assertEqual(curl.perform.call_count, 1) + + # Try a third time + os.remove(outfile) + lc.download(name, filename, hash, outfile) + self.assertEqual(curl.perform.call_count, 2) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_download_kwargs(self, mock_curl): + def mock_getinfo(info): + return 200 if info == pycurl.RESPONSE_CODE else 0 + + def mock_perform(): + with open(self.filename, 'rb') as f: + curlopts[pycurl.WRITEDATA].write(f.read()) + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.side_effect = mock_getinfo + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + with open(self.filename, 'wb') as f: + f.write(b'content') + + name = 'pyrpkg' + filename = 'pyrpkg-0.0.tar.xz' + branch = 'f22' + hash = hashlib.sha512(b'content').hexdigest() + outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') + + path = '%(name)s/%(filename)s/%(branch)s/%(hashtype)s/%(hash)s' + full_url = 'http://example.com/%s' % ( + path % {'name': name, 'filename': filename, 'branch': branch, + 'hashtype': 'sha512', 'hash': hash}) + + lc = CGILookasideCache('sha512', 'http://example.com', '_') + + # Modify the download path, to try arbitrary kwargs + lc.download_path = path + + lc.download(name, filename, hash, outfile, hashtype='sha512', + branch=branch) + self.assertEqual(curl.perform.call_count, 1) + self.assertEqual(curlopts[pycurl.URL], full_url) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_download_corrupted(self, mock_curl): + def mock_getinfo(info): + return 200 if info == pycurl.RESPONSE_CODE else 0 + + def mock_perform(): + with open(self.filename) as f: + curlopts[pycurl.WRITEDATA].write(f.read()) + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.side_effect = mock_getinfo + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + with open(self.filename, 'wb') as f: + f.write(b'content') + + hash = "not the right hash" + outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') + + lc = CGILookasideCache('sha512', 'http://example.com', '_') + self.assertRaises(DownloadError, lc.download, 'pyrpkg', + 'pyrpkg-0.0.tar.xz', hash, outfile) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_download_failed(self, mock_curl): + curl = mock_curl.return_value + curl.perform.side_effect = Exception( + 'Could not resolve host: example.com') + + with open(self.filename, 'wb') as f: + f.write(b'content') + + hash = hashlib.sha512(b'content').hexdigest() + outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') + + lc = CGILookasideCache('sha512', 'http://example.com', '_') + self.assertRaises(DownloadError, lc.download, 'pyrpkg', + 'pyrpkg-0.0.tar.xz', hash, outfile) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_download_failed_status_code(self, mock_curl): + def mock_getinfo(info): + return 500 if info == pycurl.RESPONSE_CODE else 0 + + def mock_perform(): + with open(self.filename) as f: + curlopts[pycurl.WRITEDATA].write(f.read()) + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.side_effect = mock_getinfo + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + with open(self.filename, 'wb') as f: + f.write(b'content') + + hash = hashlib.sha512(b'content').hexdigest() + outfile = os.path.join(self.workdir, 'pyrpkg-0.0.tar.xz') + + lc = CGILookasideCache('sha512', 'http://example.com', '_') + self.assertRaises(DownloadError, lc.download, 'pyrpkg', + 'pyrpkg-0.0.tar.xz', hash, outfile) + + @mock.patch('pyrpkg.lookaside.sys.stdout') + def test_print_download_progress(self, mock_stdout): + def mock_write(msg): + written_lines.append(msg) + + written_lines = [] + expected_lines = [ + '\r################## 25.0%', # nopep8 + '\r#################################### 50.0%', # nopep8 + '\r###################################################### 75.0%', # nopep8 + '\r######################################################################## 100.0%', # nopep8 + ] + + mock_stdout.write.side_effect = mock_write + + lc = CGILookasideCache('_', '_', '_') + lc.print_progress(2000.0, 500.0, 0.0, 0.0) + self.assertEqual(mock_stdout.write.call_count, 1) + self.assertEqual(len(written_lines), 1) + + lc.print_progress(2000.0, 1000.0, 0.0, 0.0) + self.assertEqual(mock_stdout.write.call_count, 2) + self.assertEqual(len(written_lines), 2) + + lc.print_progress(2000.0, 1500.0, 0.0, 0.0) + self.assertEqual(mock_stdout.write.call_count, 3) + self.assertEqual(len(written_lines), 3) + + lc.print_progress(2000.0, 2000.0, 0.0, 0.0) + self.assertEqual(mock_stdout.write.call_count, 4) + self.assertEqual(len(written_lines), 4) + + self.assertEqual(written_lines, expected_lines) + + @mock.patch('pyrpkg.lookaside.sys.stdout') + def test_print_upload_progress(self, mock_stdout): + def mock_write(msg): + written_lines.append(msg) + + written_lines = [] + expected_lines = [ + '\r################## 25.0%', # nopep8 + '\r#################################### 50.0%', # nopep8 + '\r###################################################### 75.0%', # nopep8 + '\r######################################################################## 100.0%', # nopep8 + ] + + mock_stdout.write.side_effect = mock_write + + lc = CGILookasideCache('_', '_', '_') + lc.print_progress(0.0, 0.0, 2000.0, 500.0) + self.assertEqual(mock_stdout.write.call_count, 1) + self.assertEqual(len(written_lines), 1) + + lc.print_progress(0.0, 0.0, 2000.0, 1000.0) + self.assertEqual(mock_stdout.write.call_count, 2) + self.assertEqual(len(written_lines), 2) + + lc.print_progress(0.0, 0.0, 2000.0, 1500.0) + self.assertEqual(mock_stdout.write.call_count, 3) + self.assertEqual(len(written_lines), 3) + + lc.print_progress(0.0, 0.0, 2000.0, 2000.0) + self.assertEqual(mock_stdout.write.call_count, 4) + self.assertEqual(len(written_lines), 4) + + self.assertEqual(written_lines, expected_lines) + + @mock.patch('pyrpkg.lookaside.sys.stdout') + def test_print_no_progress(self, mock_stdout): + def mock_write(msg): + written_lines.append(msg) + + written_lines = [] + + mock_stdout.write.side_effect = mock_write + + lc = CGILookasideCache('_', '_', '_') + lc.print_progress(0.0, 0.0, 0.0, 0.0) + self.assertEqual(len(written_lines), 0) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_exists(self, mock_curl): + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION](b'Available') + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + lc = CGILookasideCache('_', '_', '_') + exists = lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') + self.assertTrue(exists) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_does_not_exist(self, mock_curl): + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION](b'Missing') + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + lc = CGILookasideCache('_', '_', '_') + exists = lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') + self.assertFalse(exists) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_exists_with_custom_certs(self, mock_curl): + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION](b'Available') + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + client_cert = os.path.join(self.workdir, 'my-client-cert.cert') + with open(client_cert, 'w'): + pass + + ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') + with open(ca_cert, 'w'): + pass + + lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, + ca_cert=ca_cert) + lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') + self.assertEqual(curlopts[pycurl.SSLCERT], client_cert) + self.assertEqual(curlopts[pycurl.CAINFO], ca_cert) + + @mock.patch('pyrpkg.lookaside.logging.getLogger') + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_exists_missing_custom_certs(self, mock_curl, + mock_logger): + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION](b'Available') + + def mock_setopt(opt, value): + curlopts[opt] = value + + def mock_warn(msg): + warn_messages.append(msg) + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + warn_messages = [] + log = mock_logger.return_value + log.warning.side_effect = mock_warn + + client_cert = os.path.join(self.workdir, 'my-client-cert.cert') + ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') + + lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, + ca_cert=ca_cert) + lc.remote_file_exists('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') + self.assertTrue(pycurl.SSLCERT not in curlopts) + self.assertTrue(pycurl.CAINFO not in curlopts) + self.assertEqual(len(warn_messages), 2) + self.assertTrue('Missing certificate: ' in warn_messages[0]) + self.assertTrue('Missing certificate: ' in warn_messages[1]) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_exists_check_failed(self, mock_curl): + curl = mock_curl.return_value + curl.perform.side_effect = Exception( + 'Could not resolve host: example.com') + + lc = CGILookasideCache('_', '_', '_') + self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg', + 'pyrpkg-0.tar.xz', 'thehash') + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_exists_check_failed_status_code(self, mock_curl): + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION](b'Available') + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 500 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + lc = CGILookasideCache('_', '_', '_') + self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg', + 'pyrpkg-0.0.tar.xz', 'thehash') + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_remote_file_exists_check_unexpected_error(self, mock_curl): + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION]('Something unexpected') + + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + lc = CGILookasideCache('_', '_', '_') + self.assertRaises(UploadError, lc.remote_file_exists, 'pyrpkg', + 'pyrpkg-0.tar.xz', 'thehash') + + @mock.patch('pyrpkg.lookaside.logging.getLogger') + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_upload(self, mock_curl, mock_logger): + def mock_setopt(opt, value): + curlopts[opt] = value + + def mock_perform(): + curlopts[pycurl.WRITEFUNCTION](b'Some output') + + def mock_debug(msg): + debug_messages.append(msg) + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.perform.side_effect = mock_perform + curl.setopt.side_effect = mock_setopt + + debug_messages = [] + log = mock_logger.return_value + log.debug.side_effect = mock_debug + + lc = CGILookasideCache('sha512', '_', '_') + + with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): + lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', 'thehash') + + self.assertTrue(pycurl.HTTPPOST in curlopts) + self.assertEqual(curlopts[pycurl.HTTPPOST], [ + ('name', 'pyrpkg'), ('sha512sum', 'thehash'), + ('file', (pycurl.FORM_FILE, 'pyrpkg-0.0.tar.xz'))]) + + self.assertEqual(debug_messages, [b'Some output']) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_upload_already_exists(self, mock_curl): + curl = mock_curl.return_value + + lc = CGILookasideCache('_', '_', '_') + hash = 'thehash' + + with mock.patch.object(lc, 'remote_file_exists', lambda *x: True): + lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', hash) + + self.assertEqual(curl.perform.call_count, 0) + self.assertEqual(curl.setopt.call_count, 0) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_upload_with_custom_certs(self, mock_curl): + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.setopt.side_effect = mock_setopt + + client_cert = os.path.join(self.workdir, 'my-client-cert.cert') + with open(client_cert, 'w'): + pass + + ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') + with open(ca_cert, 'w'): + pass + + lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, + ca_cert=ca_cert) + + with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): + lc.upload('pyrpkg', 'pyrpkg-0.0.tar.xz', 'thehash') + + self.assertEqual(curlopts[pycurl.SSLCERT], client_cert) + self.assertEqual(curlopts[pycurl.CAINFO], ca_cert) + + @mock.patch('pyrpkg.lookaside.logging.getLogger') + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_upload_missing_custom_certs(self, mock_curl, mock_logger): + def mock_setopt(opt, value): + curlopts[opt] = value + + def mock_warn(msg): + warn_messages.append(msg) + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 200 + curl.setopt.side_effect = mock_setopt + + warn_messages = [] + log = mock_logger.return_value + log.warning.side_effect = mock_warn + + client_cert = os.path.join(self.workdir, 'my-client-cert.cert') + ca_cert = os.path.join(self.workdir, 'my-custom-cacert.cert') + + lc = CGILookasideCache('_', '_', '_', client_cert=client_cert, + ca_cert=ca_cert) + + with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): + lc.upload('pyrpkg', 'pyrpkg-0.tar.xz', 'thehash') + + self.assertTrue(pycurl.SSLCERT not in curlopts) + self.assertTrue(pycurl.CAINFO not in curlopts) + self.assertEqual(len(warn_messages), 2) + self.assertTrue('Missing certificate: ' in warn_messages[0]) + self.assertTrue('Missing certificate: ' in warn_messages[1]) + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_upload_failed(self, mock_curl): + curl = mock_curl.return_value + curl.perform.side_effect = Exception( + 'Could not resolve host: example.com') + + lc = CGILookasideCache('_', '_', '_') + + with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): + self.assertRaises(UploadError, lc.upload, 'pyrpkg', + 'pyrpkg-0.tar.xz', 'thehash') + + @mock.patch('pyrpkg.lookaside.pycurl.Curl') + def test_upload_failed_status_code(self, mock_curl): + def mock_setopt(opt, value): + curlopts[opt] = value + + curlopts = {} + curl = mock_curl.return_value + curl.getinfo.return_value = 500 + curl.setopt.side_effect = mock_setopt + + lc = CGILookasideCache('sha512', '_', '_') + + with mock.patch.object(lc, 'remote_file_exists', lambda *x: False): + self.assertRaises(UploadError, lc.upload, 'pyrpkg', + 'pyrpkg-0.tar.xz', 'thehash') diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 0000000..fafbaf9 --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,270 @@ +import os +import shutil +import tempfile +import unittest + +from pyrpkg import sources + + +class SourceFileEntryTestCase(unittest.TestCase): + def test_entry(self): + e = sources.SourceFileEntry('md5', 'afile', 'ahash') + expected = 'ahash afile\n' + self.assertEqual(str(e), expected) + + def test_bsd_style_entry(self): + e = sources.BSDSourceFileEntry('md5', 'afile', 'ahash') + expected = 'MD5 (afile) = ahash\n' + self.assertEqual(str(e), expected) + + +class SourcesFileTestCase(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='rpkg-tests.') + self.sourcesfile = os.path.join(self.workdir, self._testMethodName) + + def tearDown(self): + shutil.rmtree(self.workdir) + + def test_parse_empty_line(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + entry = s.parse_line('') + self.assertTrue(entry is None) + + def test_parse_eol_line(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + entry = s.parse_line('\n') + self.assertTrue(entry is None) + + def test_parse_whitespace_line(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + entry = s.parse_line(' \n') + self.assertTrue(entry is None) + + def test_parse_old_style_line(self): + s = sources.SourcesFile(self.sourcesfile, 'old') + + line = 'ahash afile\n' + entry = s.parse_line(line) + + self.assertTrue(isinstance(entry, sources.SourceFileEntry)) + self.assertEqual(entry.hashtype, 'md5') + self.assertEqual(entry.hash, 'ahash') + self.assertEqual(entry.file, 'afile') + self.assertEqual(str(entry), line) + + def test_migrate_old_style_line(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + line = 'ahash afile\n' + newline = 'MD5 (afile) = ahash\n' + entry = s.parse_line(line) + + self.assertTrue(isinstance(entry, sources.SourceFileEntry)) + self.assertEqual(entry.hashtype, 'md5') + self.assertEqual(entry.hash, 'ahash') + self.assertEqual(entry.file, 'afile') + self.assertEqual(str(entry), newline) + + def test_parse_entry_line(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + line = 'MD5 (afile) = ahash\n' + entry = s.parse_line(line) + + self.assertTrue(isinstance(entry, sources.SourceFileEntry)) + self.assertEqual(entry.hashtype, 'md5') + self.assertEqual(entry.hash, 'ahash') + self.assertEqual(entry.file, 'afile') + self.assertEqual(str(entry), line) + + def test_parse_wrong_lines(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + lines = ['ahash', + 'ahash ', + 'ahash afile', + 'SHA512 (afile) = ahash garbage', + 'MD5 SHA512 (afile) = ahash', + ] + + for line in lines: + def raises(): + s.parse_line(line) + + self.assertRaises(sources.MalformedLineError, raises) + + def test_open_new_file(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + self.assertEqual(len(s.entries), 0) + + def test_open_empty_file(self): + with open(self.sourcesfile, 'w') as f: + f.write('') + + s = sources.SourcesFile(self.sourcesfile, 'bsd') + self.assertEqual(len(s.entries), 0) + + def test_open_existing_file_with_old_style_lines(self): + lines = ['ahash afile\n', 'anotherhash anotherfile\n'] + newlines = ['MD5 (afile) = ahash\n', + 'MD5 (anotherfile) = anotherhash\n'] + + with open(self.sourcesfile, 'w') as f: + for line in lines: + f.write(line) + + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + for i, entry in enumerate(s.entries): + self.assertTrue(isinstance(entry, sources.SourceFileEntry)) + self.assertEqual(str(entry), newlines[i]) + + def test_open_existing_file(self): + lines = ['MD5 (afile) = ahash\n', 'MD5 (anotherfile) = anotherhash\n'] + + with open(self.sourcesfile, 'w') as f: + for line in lines: + f.write(line) + + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + for i, entry in enumerate(s.entries): + self.assertTrue(isinstance(entry, sources.SourceFileEntry)) + self.assertEqual(str(entry), lines[i]) + + def test_open_existing_file_with_mixed_lines(self): + lines = ['ahash afile\n', + 'anotherhash anotherfile\n', + 'MD5 (thirdfile) = thirdhash\n', + ] + expected = [ + 'MD5 (afile) = ahash\n', + 'MD5 (anotherfile) = anotherhash\n', + 'MD5 (thirdfile) = thirdhash\n', + ] + + with open(self.sourcesfile, 'w') as f: + for line in lines: + f.write(line) + + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + for i, entry in enumerate(s.entries): + self.assertTrue(isinstance(entry, sources.SourceFileEntry)) + self.assertEqual(str(entry), expected[i]) + + def test_open_existing_file_with_identical_entries_old_and_new(self): + lines = ['ahash afile\n', + 'MD5 (afile) = ahash\n', + ] + + with open(self.sourcesfile, 'w') as f: + for line in lines: + f.write(line) + + s = sources.SourcesFile(self.sourcesfile, 'bsd') + + self.assertEqual(len(s.entries), 1) + self.assertEqual(s.entries[0].hashtype, 'md5') + self.assertEqual(s.entries[0].file, 'afile') + self.assertEqual(s.entries[0].hash, 'ahash') + self.assertEqual(str(s.entries[0]), lines[-1]) + + def test_open_existing_file_with_wrong_line(self): + line = 'some garbage here\n' + + with open(self.sourcesfile, 'w') as f: + f.write(line) + + def raises(): + sources.SourcesFile(self.sourcesfile, 'bsd') + + self.assertRaises(sources.MalformedLineError, raises) + + def test_add_entry(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + self.assertEqual(len(s.entries), 0) + + s.add_entry('md5', 'afile', 'ahash') + self.assertEqual(len(s.entries), 1) + self.assertEqual(str(s.entries[-1]), 'MD5 (afile) = ahash\n') + + s.add_entry('md5', 'anotherfile', 'anotherhash') + self.assertEqual(len(s.entries), 2) + self.assertEqual(str(s.entries[-1]), 'MD5 (anotherfile) = anotherhash\n') + + def test_add_entry_twice(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + self.assertEqual(len(s.entries), 0) + + s.add_entry('md5', 'afile', 'ahash') + self.assertEqual(len(s.entries), 1) + self.assertEqual(str(s.entries[-1]), 'MD5 (afile) = ahash\n') + + s.add_entry('md5', 'afile', 'ahash') + self.assertEqual(len(s.entries), 1) + + def test_add_entry_mixing_hashtypes(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + self.assertEqual(len(s.entries), 0) + + s.add_entry('md5', 'afile', 'ahash') + self.assertEqual(len(s.entries), 1) + self.assertEqual(str(s.entries[-1]), 'MD5 (afile) = ahash\n') + + def raises(): + s.add_entry('sha512', 'anotherfile', 'anotherhash') + + self.assertRaises(sources.HashtypeMixingError, raises) + + def test_write_new_file(self): + s = sources.SourcesFile(self.sourcesfile, 'bsd') + self.assertEqual(len(s.entries), 0) + + s.add_entry('md5', 'afile', 'ahash') + s.add_entry('md5', 'anotherfile', 'anotherhash') + s.write() + + with open(self.sourcesfile) as f: + lines = f.readlines() + + self.assertEqual(len(lines), 2) + self.assertEqual(lines[0], 'MD5 (afile) = ahash\n') + self.assertEqual(lines[1], 'MD5 (anotherfile) = anotherhash\n') + + def test_write_adding_a_line(self): + lines = ['ahash afile\n', 'anotherhash anotherfile\n'] + + with open(self.sourcesfile, 'w') as f: + for line in lines: + f.write(line) + + s = sources.SourcesFile(self.sourcesfile, 'bsd') + s.add_entry('md5', 'thirdfile', 'thirdhash') + s.write() + + with open(self.sourcesfile) as f: + lines = f.readlines() + + self.assertEqual(len(lines), 3) + self.assertEqual(lines[0], 'MD5 (afile) = ahash\n') + self.assertEqual(lines[1], 'MD5 (anotherfile) = anotherhash\n') + self.assertEqual(lines[2], 'MD5 (thirdfile) = thirdhash\n') + + def test_write_over(self): + lines = ['ahash afile\n', 'anotherhash anotherfile\n'] + + with open(self.sourcesfile, 'w') as f: + for line in lines: + f.write(line) + + s = sources.SourcesFile(self.sourcesfile, 'bsd', replace=True) + s.add_entry('md5', 'thirdfile', 'thirdhash') + s.write() + + with open(self.sourcesfile) as f: + lines = f.readlines() + + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0], 'MD5 (thirdfile) = thirdhash\n') diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..96f6bdf --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,202 @@ +import unittest +import warnings + +import mock + +from pyrpkg.utils import cached_property, warn_deprecated, log_result + + +class CachedPropertyTestCase(unittest.TestCase): + def test_computed_only_once(self): + class Foo(object): + @cached_property + def foo(self): + runs.append("run once") + return 42 + + runs = [] + + f = Foo() + self.assertEqual(len(runs), 0) + self.assertEqual(f.foo, 42) + self.assertEqual(len(runs), 1) + self.assertEqual(f.foo, 42) + self.assertEqual(len(runs), 1) + + def test_not_shared_between_properties(self): + class Foo(object): + @cached_property + def foo(self): + foo_runs.append("run once") + return 42 + + @cached_property + def bar(self): + bar_runs.append("run once") + return 43 + + foo_runs = [] + bar_runs = [] + + f = Foo() + self.assertEqual(len(foo_runs), 0) + self.assertEqual(f.foo, 42) + self.assertEqual(len(foo_runs), 1) + self.assertEqual(f.foo, 42) + self.assertEqual(len(foo_runs), 1) + + self.assertEqual(len(bar_runs), 0) + self.assertEqual(f.bar, 43) + self.assertEqual(len(bar_runs), 1) + self.assertEqual(f.bar, 43) + self.assertEqual(len(bar_runs), 1) + + def test_not_shared_between_instances(self): + class Foo(object): + @cached_property + def foo(self): + foo_runs.append("run once") + return 42 + + class Bar(object): + @cached_property + def foo(self): + bar_runs.append("run once") + return 43 + + foo_runs = [] + bar_runs = [] + + f = Foo() + self.assertEqual(len(foo_runs), 0) + self.assertEqual(f.foo, 42) + self.assertEqual(len(foo_runs), 1) + self.assertEqual(f.foo, 42) + self.assertEqual(len(foo_runs), 1) + + b = Bar() + self.assertEqual(len(bar_runs), 0) + self.assertEqual(b.foo, 43) + self.assertEqual(len(bar_runs), 1) + self.assertEqual(b.foo, 43) + self.assertEqual(len(bar_runs), 1) + + def test_not_shared_when_inheriting(self): + class Foo(object): + @cached_property + def foo(self): + foo_runs.append("run once") + return 42 + + class Bar(Foo): + @cached_property + def foo(self): + bar_runs.append("run once") + return 43 + + foo_runs = [] + bar_runs = [] + + b = Bar() + self.assertEqual(len(bar_runs), 0) + self.assertEqual(b.foo, 43) + self.assertEqual(len(bar_runs), 1) + self.assertEqual(b.foo, 43) + self.assertEqual(len(bar_runs), 1) + + f = Foo() + self.assertEqual(len(foo_runs), 0) + self.assertEqual(f.foo, 42) + self.assertEqual(len(foo_runs), 1) + self.assertEqual(f.foo, 42) + self.assertEqual(len(foo_runs), 1) + + bar_runs = [] + b = Bar() + self.assertEqual(len(bar_runs), 0) + self.assertEqual(b.foo, 43) + self.assertEqual(len(bar_runs), 1) + self.assertEqual(b.foo, 43) + self.assertEqual(len(bar_runs), 1) + + +class DeprecationUtilsTestCase(unittest.TestCase): + def setUp(self): + warnings.simplefilter('always', DeprecationWarning) + + @mock.patch('sys.stderr') + def test_warn_deprecated(self, mock_stderr): + class Foo(object): + def old_method(self): + warn_deprecated(self.__class__.__name__, 'old_method', + 'new_method') + return self.new_method() + + def new_method(self): + return "Yay!" + + def mock_write(msg): + written_lines.append(msg) + + written_lines = [] + mock_stderr.write.side_effect = mock_write + + foo = Foo() + self.assertEqual(foo.old_method(), foo.new_method()) + self.assertEqual(len(written_lines), 1) + self.assertTrue('DeprecationWarning' in written_lines[0]) + self.assertTrue('Foo.old_method' in written_lines[0]) + self.assertTrue('Foo.new_method' in written_lines[0]) + + warnings.simplefilter('error', DeprecationWarning) + self.assertRaises(DeprecationWarning, foo.old_method) + self.assertEqual(len(written_lines), 1) + + +class LogResultTestCase(unittest.TestCase): + def setUp(self): + self.logs = [] + + def info(msg): + self.logs.append(msg) + + self.log_func = info + + def test_dict_result(self): + obj = {'spam': 'maps'} + expected = [ + 'spam:', + ' maps', + ] + log_result(self.log_func, obj) + self.assertEqual(self.logs, expected) + + def test_list_result(self): + obj = ['eggs', 'bacon', 'hash'] + expected = [ + 'eggs', + 'bacon', + 'hash', + ] + log_result(self.log_func, obj) + self.assertEqual(self.logs, expected) + + def test_str_result(self): + obj = 'spam' + expected = [ + 'spam', + ] + log_result(self.log_func, obj) + self.assertEqual(self.logs, expected) + + def test_complex_result(self): + obj = {'breakfast': ['eggs', 'bacon', {'spam': 'maps'}]} + expected = [ + 'breakfast:', + ' eggs', + ' bacon', + ' spam:', + ' maps', + ] + log_result(self.log_func, obj) + self.assertEqual(self.logs, expected)