From ddf32cfedc2a76d89b752266fcc79000d826d217 Mon Sep 17 00:00:00 2001 From: Stef Walter Date: Jan 11 2019 12:42:55 +0000 Subject: Fix standard-test-source to work on Fedora or RHEL Use the rather standard HTTP API for the sources file and the dist-git lookaside cache to retrieve sources. Don't rely on the fedpkg or rhpkg tooling because these are unnecessarily diverged between Fedora and RHEL Solves Issue #271 --- diff --git a/roles/standard-test-source/library/README.md b/roles/standard-test-source/library/README.md new file mode 100644 index 0000000..b67d131 --- /dev/null +++ b/roles/standard-test-source/library/README.md @@ -0,0 +1,21 @@ +Modules for this role + +## source-lookaside.py + +This module is only used by the standard-test-source role. But further documentation +about how to call it is availeble in the module itself. + +To hack on a module, create an args.json file like this: + + { + "ANSIBLE_MODULE_ARGS": { + "package": "cockpit", + "target": "extracted" + } + } + +Then call the module like this or this, depending on your Ansible version: + + $ python2 roles/standard-test-source/library/source-lookaside.py args.json + $ python3 roles/standard-test-source/library/source-lookaside.py args.json + diff --git a/roles/standard-test-source/library/source-lookaside.py b/roles/standard-test-source/library/source-lookaside.py new file mode 100644 index 0000000..5315b73 --- /dev/null +++ b/roles/standard-test-source/library/source-lookaside.py @@ -0,0 +1,234 @@ +#!/usr/bin/python3 +# This code runs under python2 and python3 + +# +# Copyright: (c) 2018, Stef Walter +# +# The MIT License (MIT) +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from ansible.module_utils.basic import AnsibleModule + +import errno +import logging +import glob +import re +import os +import shutil + +try: + from urllib.request import urlopen + from urllib.error import URLError + from configparser import ConfigParser +except ImportError: + from urllib2 import urlopen, URLError + from ConfigParser import ConfigParser + +ANSIBLE_METADATA = { + 'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community' +} + +DOCUMENTATION = ''' +--- +module: source-lookaside + +short_description: Extract source code from Fedora or RHEL lookaside dist-git caches + +version_added: "2.4" + +description: + - "This module retrives source code artifacts from the Fedora or RHEL dist-git lookaside caches" + +options: + package: + description: + - The package name + required: true + sources: + description: + - The sources file that includes hashes. Default: "./sources" + required: false + target: + description: + - Target directory to write sources to. Default: same directory as sources + required: false + +author: + - Stef Walter (@stefwalter) +''' + +EXAMPLES = ''' +# Pull source tarball +- name: Pull sources + source-lookaside: + package: cockpit + +# Pull a sources from the given sources file +- name: Pull sources + source-lookaside: + package: cockpit + sources: /path/to/cockpit/sources + +# Pull the sources into to a specific directory +- name: Pull sources + source-lookaside: + package: cockpit + target: /path/to/source +''' + +RETURN = ''' +original_sources: + description: The path to the sources file +sources: + description: A list of source artifact files +''' + +LOOKASIDES = ( + "https://src.fedoraproject.org/repo/pkgs", +) + +LOOKASIDE_URI = "/rpms/{name}/{filename}/{hashtype}/{hash}/{filename}" + +# Location to find more lookaside URLs +LOOKASIDE_CONFIG = "/etc/rpkg/*.conf" + +PATTERNS = ( + re.compile(r'^(?P[^ ]+?) \((?P[^ )]+?)\) = (?P[^ ]+?)$'), + re.compile(r'^(?P[^ ]+?) (?P[^ )]+?)$'), +) + +# A logger to write to +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +# Because some people have other lookaside URLs configured, look them up +def lookasides(): + for url in LOOKASIDES: + yield url + LOOKASIDE_URI + config = ConfigParser() + config.read(glob.glob(LOOKASIDE_CONFIG)) + for section in config.sections(): + if config.has_option(section, "lookaside"): + yield config.get(section, "lookaside") + LOOKASIDE_URI + + +# Parses the sources file into a list of possible urls to retrieve +def urls(package, sources): + if not os.path.exists(sources): + return + with open(sources, 'r') as fp: + for line in fp.readlines(): + for pattern in PATTERNS: + match = pattern.match(line.strip()) + if match is None: + continue + fields = match.groupdict() + fields['hashtype'] = fields.get('hashtype', 'md5').lower() + fields['name'] = package + yield tuple(map(lambda url: url.format(**fields), lookasides())) + + +def mkdirs(directory): + try: + os.makedirs(directory) + except OSError as ex: + if ex.errno == errno.EEXIST and os.path.isdir(directory): + pass + + +def retrieve(url, target): + name = os.path.basename(url) + dest = os.path.join(target, name) + + mkdirs(target) + + try: + with open(dest, 'wb') as fp: + shutil.copyfileobj(urlopen(url), fp) + except URLError as ex: + if not hasattr(ex, "code") or ex.code != 404: + logger.error("{0}: {1} {2}".format(name, url, str(ex))) + return None + + return dest + + +def run_module(): + # define available arguments/parameters a user can pass to the module + module_args = dict( + package=dict(type='str', required=True), + sources=dict(type='str', required=False, default="./sources"), + target=dict(type='str', required=False, default=""), + ) + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + + package = module.params['package'].strip() + sources = module.params['sources'] + target = module.params['target'] + + # Default target to same directory as sources file + if not target: + target = os.path.dirname(sources) + + # The results + result = dict(changed=False, original_sources=sources, sources=[]) + + if not os.path.exists(sources): + module.fail_json(msg='The sources file does not exist: {0}'.format(sources), **result) + return + + # if the user is working with this module in only check mode we do not + # want to make any changes to the environment, just return the current + # state with no modifications + if module.check_mode: + return result + + # Get a list of possible URLs for each of the sources + for possible in urls(package, sources): + name = None + for url in possible: + name = os.path.basename(url) + logger.info("{0}: {1}\n".format(name, url)) + + # Try to retrieve the possible url for this source + dest = retrieve(url, target) + if dest: + result['changed'] = True + result['sources'].append(dest) + break + + # We fail module execution if we cannot retrieve each file + else: + module.fail_json(msg="Unable to retrieve source file: {}".format(name), **result) + + # Successful module execution + module.exit_json(**result) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/roles/standard-test-source/tasks/main.yml b/roles/standard-test-source/tasks/main.yml index 2be973e..53ab0e6 100644 --- a/roles/standard-test-source/tasks/main.yml +++ b/roles/standard-test-source/tasks/main.yml @@ -13,14 +13,24 @@ setup: delegate_facts: True - # The dist doesn't actually matter here - - name: Download sources, extract, and strip leading path + - name: Get the specfile package name + shell: rpm -q --specfile --queryformat="%{NAME}\n" {{pkgdir}}/*.spec | head -n1 + args: + warn: false + register: name + + - name: Pull down the source tarballs + source-lookaside: + package: "{{name.stdout}}" + sources: "{{pkgdir}}/sources" + target: "{{pkgdir}}/" + + - name: Extract and setup the sources shell: | - set -e - rm -rf {{ srcdir }} - fedpkg --release=master prep --builddir={{ srcdir }} + rm -rf "{{srcdir}}" + rpmbuild -bp {{pkgdir}}/*.spec --nodeps --define "_sourcedir {{pkgdir}}/" --define "_builddir {{srcdir}}" args: - chdir: "{{playbook_dir}}/.." + warn: false - name: Flatten sources shell: | diff --git a/tests/test-source-zsh/zsh.spec b/tests/test-source-zsh/zsh.spec index c710d4d..75ad157 100644 --- a/tests/test-source-zsh/zsh.spec +++ b/tests/test-source-zsh/zsh.spec @@ -1,4 +1,4 @@ -Summary: Powerful interactive shell +Summary: Spec file for zsh to test lookaside cache Name: zsh Version: 5.6.2 Release: 3%{?dist} @@ -158,286 +158,3 @@ fi %changelog * Fri Nov 30 2018 Kamil Dudka - 5.6.2-3 - return non-zero exit status on nested parse error (#1654989) - -* Mon Nov 12 2018 Kamil Dudka - 5.6.2-2 -- fix programming mistakes detected by static analysis - -* Fri Sep 14 2018 Kamil Dudka - 5.6.2-1 -- update to latest upstream release - -* Mon Sep 10 2018 Kamil Dudka - 5.6.1-1 -- update to latest upstream release - -* Tue Sep 04 2018 Kamil Dudka - 5.6-1 -- update to latest upstream release (fixes CVE-2018-0502 and CVE-2018-13259) - -* Sat Jul 14 2018 Fedora Release Engineering - 5.5.1-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild - -* Tue Apr 17 2018 Kamil Dudka - 5.5.1-1 -- update to latest upstream release - -* Mon Apr 09 2018 Kamil Dudka - 5.5-1 -- update to latest upstream release, which fixes the following vulnerabilities: - CVE-2018-1100 - stack-based buffer overflow in utils.c:checkmailpath() - CVE-2018-1083 - stack-based buffer overflow in compctl.c:gen_matches_files() - CVE-2018-1071 - stack-based buffer overflow in exec.c:hashcmd() - -* Tue Mar 06 2018 Kamil Dudka - 5.4.2-7 -- avoid crash when copying empty hash table (CVE-2018-7549) -- avoid NULL dereference when using ${(PA)...} on an empty array (CVE-2018-7548) - -* Mon Feb 19 2018 Kamil Dudka - 5.4.2-6 -- add explicit BR for the gcc compiler - -* Fri Feb 09 2018 Fedora Release Engineering - 5.4.2-5 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild - -* Mon Jan 22 2018 Kamil Dudka - 5.4.2-4 -- make build of run-time loadable modules work again (#1535422) - -* Tue Jan 16 2018 Kamil Dudka - 5.4.2-3 -- rebuild against latest gdbm-devel (#1533176) - -* Wed Oct 04 2017 Kamil Dudka - 5.4.2-2 -- make the call depth limit configurable by $FUNCNEST (#1441092) - -* Mon Aug 28 2017 Kamil Dudka - 5.4.2-1 -- update to latest upstream release - -* Wed Aug 09 2017 Kamil Dudka - 5.4.1-1 -- update to latest upstream release - -* Tue Aug 01 2017 Kamil Dudka - 5.3.1-12 -- use %%make_install instead of %%makeinstall, which is deprecated -- modernize spec file (Group tag, %%clean, %%defattr) - -* Thu Jul 27 2017 Fedora Release Engineering - 5.3.1-11 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild - -* Fri Jul 07 2017 Kamil Dudka - 5.3.1-10 -- enable parallel build - -* Wed Jun 14 2017 Kamil Dudka - 5.3.1-9 -- fix unsafe use of a static buffer in history isearch (#1461483) - -* Thu Jun 08 2017 Kamil Dudka - 5.3.1-8 -- make the zsh-html subpackage noarch (#1459657) - -* Thu May 25 2017 Kamil Dudka - 5.3.1-7 -- drop unmaintained and undocumented zshprompt.pl script - -* Wed May 17 2017 Kamil Dudka - 5.3.1-6 -- drop workaround for broken terminals over serial port (#56353) - -* Thu May 11 2017 Kamil Dudka - 5.3.1-5 -- compile with -fconserve-stack to prevent stack overflow (#1441092) - -* Fri Mar 31 2017 Jason L Tibbitts III - 5.3.1-4 -- Add build deps on gdbm-devel and pcre-devel. Pass --enable-pcre to - configure. These should ensure that the pcre and gdbm modules are built. - (#1438009) - -* Sat Feb 11 2017 Fedora Release Engineering - 5.3.1-3 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild - -* Thu Dec 22 2016 Kamil Dudka - 5.3.1-2 -- do not require the hostname package when being built on RHEL-6 - -* Wed Dec 21 2016 Kamil Dudka - 5.3.1-1 -- Update to latest upstream release: Zsh 5.3.1 - -* Wed Dec 14 2016 Kamil Dudka - 5.3-2 -- drop zsh-4.3.6-8bit-prompts.patch which was superseeded by an upstream patch - (see http://www.zsh.org/mla/users/2007/msg00468.html for details) -- drop undocumented zsh-test-C02-dev_fd-mock.patch - -* Tue Dec 13 2016 Kamil Dudka - 5.3-1 -- apply patches automatically to ease maintenance -- Update to latest upstream release: Zsh 5.3 - -* Fri Feb 05 2016 Fedora Release Engineering - 5.2-5 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_24_Mass_Rebuild - -* Wed Jan 27 2016 Kamil Dudka - 5.2-4 -- prevent zsh from crashing when printing the "out of memory" message (#1300958) - -* Thu Jan 07 2016 Jason L Tibbitts III - 5.2-3 -- Add patch to fix VCS_INFO_nbvsformats bug. - -* Fri Dec 25 2015 Adrien Vergé - 5.2-2 -- update zsh completion script for dnf to the latest upstream version - -* Thu Dec 03 2015 Kamil Dudka - 5.2-1 -- Update to latest upstream release: Zsh 5.2 - -* Thu Nov 05 2015 Kamil Dudka - 5.1.1-3 -- make loading of module's dependencies work again (#1277996) - -* Thu Oct 08 2015 Kamil Dudka - 5.1.1-2 -- fix crash in ksh mode with -n and $HOME (#1269883) - -* Mon Sep 14 2015 Kamil Dudka - 5.1.1-1 -- Update to latest upstream release: Zsh 5.1.1 - -* Mon Aug 31 2015 Kamil Dudka - 5.1-1 -- Update to latest upstream release: Zsh 5.1 -- remove outdated workarounds in %%check - -* Thu Jul 30 2015 Kamil Dudka - 5.0.8-6 -- fix handling of command substitution in math context - -* Wed Jul 22 2015 Kamil Dudka - 5.0.8-5 -- prevent infinite recursion in ihungetc() (#1245712) - -* Tue Jul 07 2015 Kamil Dudka - 5.0.8-4 -- backport completion for dnf (#1239337) - -* Thu Jul 02 2015 Kamil Dudka - 5.0.8-3 -- backport completion-related upstream fixes (#1238544) - -* Fri Jun 19 2015 Fedora Release Engineering - 5.0.8-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_23_Mass_Rebuild - -* Wed Jun 03 2015 Kamil Dudka - 5.0.8-1 -- Update to latest upstream release: Zsh 5.0.8 - -* Fri May 22 2015 Kamil Dudka - 5.0.7-8 -- fix SIGSEGV of the syntax check in ksh emulation mode (#1222867) - -* Mon Apr 20 2015 Kamil Dudka - 5.0.7-7 -- fix SIGSEGV when handling heredocs and keyboard interrupt (#972624) -- queue signals when manipulating global state to avoid deadlock - -* Sun Jan 25 2015 Kamil Dudka - 5.0.7-6 -- use correct allocation function in the new 'cd' code (#1183238) - -* Fri Jan 23 2015 Kamil Dudka - 5.0.7-5 -- suppress a warning about closing an already closed file descriptor (#1184002) -- improve handling of NULL in the 'cd' built-in (#1183238) - -* Wed Nov 19 2014 Kamil Dudka - 5.0.7-4 -- update documentation of POSIX_JOBS in the zshoptions.1 man page (#1162198) - -* Tue Nov 18 2014 Kamil Dudka - 5.0.7-3 -- replace an incorrect comment in /etc/zshenv (#1164313) - -* Mon Nov 10 2014 Kamil Dudka - 5.0.7-2 -- make the wait built-in work for already exited processes (#1162198) - -* Wed Oct 08 2014 Dominic Hopf - 5.0.7-1 -- Update to latest upstream release: Zsh 5.0.7 - -* Thu Aug 28 2014 Dominic Hopf - 5.0.6-1 -- Update to latest upstream release: Zsh 5.0.6 - -* Mon Aug 18 2014 Fedora Release Engineering - 5.0.5-8 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_22_Mass_Rebuild - -* Thu Jul 17 2014 Dominic Hopf - 5.0.5-7 -- apply upstream patch which fixes CPU load issue (RHBZ#1120424) - -* Wed Jul 09 2014 Adam Jackson 5.0.5-6 -- Fix missing 'fi' in %%post - -* Thu Jul 03 2014 Dominic Hopf - 5.0.5-5 -- improve handling of /etc/shells - -* Wed Jul 02 2014 Dominic Hopf - 5.0.5-4 -- fix FTBFS issue (RHBZ#1106713) -- remove individual _bindir setting; install to /usr/bin/ (RHBZ#1034060) -- require info package instead of /sbin/install-info binary - -* Sat Jun 07 2014 Fedora Release Engineering - 5.0.5-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_21_Mass_Rebuild - -* Tue Apr 08 2014 Dominic Hopf - 5.0.5-1 -- Update to latest upstream release: Zsh 5.0.5 - -* Thu Jan 16 2014 James Antill - 5.0.2-8 -- Remove unneeded build require on tetex. - -* Sat Oct 26 2013 Dominic Hopf - 5.0.2-7 -- Require hostname package instead of /bin/hostname - -* Tue Oct 22 2013 Dominic Hopf - 5.0.2-6 -- remove systemd completion, it delivers it's own now (RHBZ#1022039) - -* Thu Aug 01 2013 Dominic Hopf - 5.0.2-5 -- update systemd completion (adds machinectl command) - -* Tue Jun 25 2013 Dominic Hopf - 5.0.2-4 -- up-to-date systemd completion (#949003) -- apply patch for building for aarch64 (#926864) - -* Mon Apr 15 2013 James Antill - 5.0.2-3 -- Fix the changelog dates. -- Fix the texi itemx bug. -- Resolves: bug#927863 - -* Fri Feb 15 2013 Fedora Release Engineering - 5.0.2-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_19_Mass_Rebuild - -* Tue Jan 08 2013 Dominic Hopf - 5.0.2-1 -- Update to new upstream version: Zsh 5.0.2 - -* Wed Nov 21 2012 Dominic Hopf - 5.0.0-1 -- Update to new upstream version: Zsh 5.0.0 - -* Sun Jul 22 2012 Fedora Release Engineering - 4.3.17-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_18_Mass_Rebuild - -* Sun Mar 04 2012 Dominic Hopf - 4.3.17-1 -- Update to new upstream version: Zsh 4.3.17 - -* Sat Jan 14 2012 Fedora Release Engineering - 4.3.15-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_17_Mass_Rebuild - -* Sat Dec 24 2011 Dominic Hopf - 4.3.15-1 -- Update to new upstream version: Zsh 4.3.15 - -* Sat Dec 17 2011 Dominic Hopf - 4.3.14-2 -- change the License field to MIT (RHBZ#768548) - -* Sat Dec 10 2011 Dominic Hopf - 4.3.14-1 -- Update to new upstream version: Zsh 4.3.14 - -* Sat Dec 03 2011 Dominic Hopf - 4.3.13-1 -- Update to new upstream version: Zsh 4.3.13 - -* Sat Aug 13 2011 Dominic Hopf - 4.3.12-1 -- Update to new upstream version: Zsh 4.3.12 - -* Tue Feb 08 2011 Fedora Release Engineering - 4.3.11-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_15_Mass_Rebuild - -* Thu Jan 20 2011 Christopher Ailon - 4.3.11-1 -- Rebase to upstream version 4.3.11 - -* Tue Dec 7 2010 Toshio Kuratomi - 4.3.10-6 -- Rebuild for FTBFS https://bugzilla.redhat.com/show_bug.cgi?id=631197 -- Remove deprecated PreReq, the packages aren't needed at runtime and they're - already in Requires(post,preun,etc): lines. - -* Mon Mar 22 2010 James Antill - 4.3.10-5 -- Add pathmunge to our /etc/zshrc, for profile.d compat. -- Resolves: bug#548960 - -* Fri Aug 7 2009 James Antill - 4.3.10-4 -- Allow --excludedocs command to work! -- Resolves: bug#515986 - -* Mon Jul 27 2009 Fedora Release Engineering - 4.3.10-3 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_12_Mass_Rebuild - -* Mon Jul 20 2009 James Antill - 4.3.10-1 -- Import new upstream 4.3.10 - -* Wed Jun 10 2009 Karsten Hopp 4.3.9-4.1 -- skip D02glob test on s390, too - -* Mon Mar 2 2009 James Antill - 4.3.9-4 -- Remove D02glob testcase on ppc/ppc64, and hope noone cares - -* Wed Feb 25 2009 Fedora Release Engineering - 4.3.9-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild