From 806aa1a0da7b40b3a15f7557588c56e61f9768b1 Mon Sep 17 00:00:00 2001
From: Andrei Stepanov <astepano@redhat.com>
Date: Jan 17 2019 15:58:17 +0000
Subject: Merge #293 `Fix standard-test-source to work on Fedora or RHEL`


---

diff --git a/roles/standard-test-source/library/README.md b/roles/standard-test-source/library/README.md
new file mode 100644
index 0000000..b67d131
--- /dev/null
+++ b/roles/standard-test-source/library/README.md
@@ -0,0 +1,21 @@
+Modules for this role
+
+## source-lookaside.py
+
+This module is only used by the standard-test-source role. But further documentation
+about how to call it is availeble in the module itself.
+
+To hack on a module, create an args.json file like this:
+
+    {
+        "ANSIBLE_MODULE_ARGS": {
+            "package": "cockpit",
+            "target": "extracted"
+        }
+    }
+
+Then call the module like this or this, depending on your Ansible version:
+
+    $ python2 roles/standard-test-source/library/source-lookaside.py args.json
+    $ python3 roles/standard-test-source/library/source-lookaside.py args.json
+
diff --git a/roles/standard-test-source/library/source-lookaside.py b/roles/standard-test-source/library/source-lookaside.py
new file mode 100644
index 0000000..5315b73
--- /dev/null
+++ b/roles/standard-test-source/library/source-lookaside.py
@@ -0,0 +1,234 @@
+#!/usr/bin/python3
+# This code runs under python2 and python3
+
+#
+# Copyright: (c) 2018, Stef Walter <stefw@redhat.com>
+#
+# The MIT License (MIT)
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+from ansible.module_utils.basic import AnsibleModule
+
+import errno
+import logging
+import glob
+import re
+import os
+import shutil
+
+try:
+    from urllib.request import urlopen
+    from urllib.error import URLError
+    from configparser import ConfigParser
+except ImportError:
+    from urllib2 import urlopen, URLError
+    from ConfigParser import ConfigParser
+
+ANSIBLE_METADATA = {
+    'metadata_version': '1.1',
+    'status': ['preview'],
+    'supported_by': 'community'
+}
+
+DOCUMENTATION = '''
+---
+module: source-lookaside
+
+short_description: Extract source code from Fedora or RHEL lookaside dist-git caches
+
+version_added: "2.4"
+
+description:
+    - "This module retrives source code artifacts from the Fedora or RHEL dist-git lookaside caches"
+
+options:
+    package:
+        description:
+            - The package name
+        required: true
+    sources:
+        description:
+            - The sources file that includes hashes. Default: "./sources"
+        required: false
+    target:
+        description:
+            - Target directory to write sources to. Default: same directory as sources
+        required: false
+
+author:
+    - Stef Walter (@stefwalter)
+'''
+
+EXAMPLES = '''
+# Pull source tarball
+- name: Pull sources
+  source-lookaside:
+    package: cockpit
+
+# Pull a sources from the given sources file
+- name: Pull sources
+  source-lookaside:
+    package: cockpit
+    sources: /path/to/cockpit/sources
+
+# Pull the sources into to a specific directory
+- name: Pull sources
+  source-lookaside:
+    package: cockpit
+    target: /path/to/source
+'''
+
+RETURN = '''
+original_sources:
+    description: The path to the sources file
+sources:
+    description: A list of source artifact files
+'''
+
+LOOKASIDES = (
+    "https://src.fedoraproject.org/repo/pkgs",
+)
+
+LOOKASIDE_URI = "/rpms/{name}/{filename}/{hashtype}/{hash}/{filename}"
+
+# Location to find more lookaside URLs
+LOOKASIDE_CONFIG = "/etc/rpkg/*.conf"
+
+PATTERNS = (
+    re.compile(r'^(?P<hashtype>[^ ]+?) \((?P<filename>[^ )]+?)\) = (?P<hash>[^ ]+?)$'),
+    re.compile(r'^(?P<hash>[^ ]+?)  (?P<filename>[^ )]+?)$'),
+)
+
+# A logger to write to
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+# Because some people have other lookaside URLs configured, look them up
+def lookasides():
+    for url in LOOKASIDES:
+        yield url + LOOKASIDE_URI
+    config = ConfigParser()
+    config.read(glob.glob(LOOKASIDE_CONFIG))
+    for section in config.sections():
+        if config.has_option(section, "lookaside"):
+            yield config.get(section, "lookaside") + LOOKASIDE_URI
+
+
+# Parses the sources file into a list of possible urls to retrieve
+def urls(package, sources):
+    if not os.path.exists(sources):
+        return
+    with open(sources, 'r') as fp:
+        for line in fp.readlines():
+            for pattern in PATTERNS:
+                match = pattern.match(line.strip())
+                if match is None:
+                    continue
+                fields = match.groupdict()
+                fields['hashtype'] = fields.get('hashtype', 'md5').lower()
+                fields['name'] = package
+                yield tuple(map(lambda url: url.format(**fields), lookasides()))
+
+
+def mkdirs(directory):
+    try:
+        os.makedirs(directory)
+    except OSError as ex:
+        if ex.errno == errno.EEXIST and os.path.isdir(directory):
+            pass
+
+
+def retrieve(url, target):
+    name = os.path.basename(url)
+    dest = os.path.join(target, name)
+
+    mkdirs(target)
+
+    try:
+        with open(dest, 'wb') as fp:
+            shutil.copyfileobj(urlopen(url), fp)
+    except URLError as ex:
+        if not hasattr(ex, "code") or ex.code != 404:
+            logger.error("{0}: {1} {2}".format(name, url, str(ex)))
+        return None
+
+    return dest
+
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        package=dict(type='str', required=True),
+        sources=dict(type='str', required=False, default="./sources"),
+        target=dict(type='str', required=False, default=""),
+    )
+
+    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+
+    package = module.params['package'].strip()
+    sources = module.params['sources']
+    target = module.params['target']
+
+    # Default target to same directory as sources file
+    if not target:
+        target = os.path.dirname(sources)
+
+    # The results
+    result = dict(changed=False, original_sources=sources, sources=[])
+
+    if not os.path.exists(sources):
+        module.fail_json(msg='The sources file does not exist: {0}'.format(sources), **result)
+        return
+
+    # if the user is working with this module in only check mode we do not
+    # want to make any changes to the environment, just return the current
+    # state with no modifications
+    if module.check_mode:
+        return result
+
+    # Get a list of possible URLs for each of the sources
+    for possible in urls(package, sources):
+        name = None
+        for url in possible:
+            name = os.path.basename(url)
+            logger.info("{0}: {1}\n".format(name, url))
+
+            # Try to retrieve the possible url for this source
+            dest = retrieve(url, target)
+            if dest:
+                result['changed'] = True
+                result['sources'].append(dest)
+                break
+
+        # We fail module execution if we cannot retrieve each file
+        else:
+            module.fail_json(msg="Unable to retrieve source file: {}".format(name), **result)
+
+    # Successful module execution
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/roles/standard-test-source/tasks/main.yml b/roles/standard-test-source/tasks/main.yml
index 2be973e..53ab0e6 100644
--- a/roles/standard-test-source/tasks/main.yml
+++ b/roles/standard-test-source/tasks/main.yml
@@ -13,14 +13,24 @@
     setup:
     delegate_facts: True
 
-  # The dist doesn't actually matter here
-  - name: Download sources, extract, and strip leading path
+  - name: Get the specfile package name
+    shell: rpm -q --specfile --queryformat="%{NAME}\n" {{pkgdir}}/*.spec | head -n1
+    args:
+      warn: false
+    register: name
+
+  - name: Pull down the source tarballs
+    source-lookaside:
+      package: "{{name.stdout}}"
+      sources: "{{pkgdir}}/sources"
+      target: "{{pkgdir}}/"
+
+  - name: Extract and setup the sources
     shell: |
-      set -e
-      rm -rf {{ srcdir }}
-      fedpkg --release=master prep --builddir={{ srcdir }}
+      rm -rf "{{srcdir}}"
+      rpmbuild -bp {{pkgdir}}/*.spec --nodeps --define "_sourcedir {{pkgdir}}/" --define "_builddir {{srcdir}}"
     args:
-      chdir: "{{playbook_dir}}/.."
+      warn: false
 
   - name: Flatten sources
     shell: |
diff --git a/roles/standard-test-source/vars/main.yml b/roles/standard-test-source/vars/main.yml
index 51a578d..6ba1825 100644
--- a/roles/standard-test-source/vars/main.yml
+++ b/roles/standard-test-source/vars/main.yml
@@ -1,3 +1,4 @@
 ---
+pkgdir: "{{ playbook_dir }}/.."
 srcdir: "{{ playbook_dir }}/source"
 flatten: True
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..67d89aa
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1,3 @@
+*.retry
+test-source-zsh/*.tar.xz
+source/
diff --git a/tests/source.yml b/tests/source.yml
new file mode 100644
index 0000000..c23901a
--- /dev/null
+++ b/tests/source.yml
@@ -0,0 +1,14 @@
+# Tests for standard-test-source role
+- hosts: localhost
+  tags:
+  - atomic
+  - classic
+  - container
+  roles:
+  - role: standard-test-source
+    pkgdir: "{{playbook_dir}}/test-source-zsh"
+    srcdir: "{{playbook_dir}}/source"
+  tasks:
+  - name: "Check if sources were pulled for zsh"
+    shell: ls "{{playbook_dir}}/source/configure" "{{playbook_dir}}/source/aczsh.m4" "{{playbook_dir}}/source/patch-created.txt"
+
diff --git a/tests/test-source-zsh/create-file.patch b/tests/test-source-zsh/create-file.patch
new file mode 100644
index 0000000..1bab726
--- /dev/null
+++ b/tests/test-source-zsh/create-file.patch
@@ -0,0 +1,21 @@
+From 701936e50d4530b64d580d3fb1e3afbcb4fc1292 Mon Sep 17 00:00:00 2001
+From: Stef Walter <stefw@redhat.com>
+Date: Thu, 10 Jan 2019 07:02:48 +0100
+Subject: [PATCH] Patch that creates a file in the sources
+
+---
+ patch-created.txt | 2 ++
+ 1 file changed, 2 insertions(+)
+ create mode 100644 patch-created.txt
+
+diff --git a/patch-created.txt b/patch-created.txt
+new file mode 100644
+index 0000000..6e6acb8
+--- /dev/null
++++ b/patch-created.txt
+@@ -0,0 +1,2 @@
++This is a file created by a patch
++It should be present in the extracted sources
+-- 
+2.20.1
+
diff --git a/tests/test-source-zsh/sources b/tests/test-source-zsh/sources
new file mode 100644
index 0000000..498d609
--- /dev/null
+++ b/tests/test-source-zsh/sources
@@ -0,0 +1 @@
+SHA512 (zsh-5.6.2.tar.xz) = f0a49e41b55eb478692ab5471d7c9828956b7e96bc82944202b0ef1c49a889b21a0e7682aa5f59fd0054ebfd866c2244c8a622e7aa46c13038af5c226c48a3a2
diff --git a/tests/test-source-zsh/zsh.spec b/tests/test-source-zsh/zsh.spec
new file mode 100644
index 0000000..75ad157
--- /dev/null
+++ b/tests/test-source-zsh/zsh.spec
@@ -0,0 +1,160 @@
+Summary: Spec file for zsh to test lookaside cache
+Name: zsh
+Version: 5.6.2
+Release: 3%{?dist}
+License: MIT
+URL: http://zsh.sourceforge.net/
+Source0: https://downloads.sourceforge.net/%{name}/%{name}-%{version}.tar.xz
+Source1: zlogin.rhs
+Source2: zlogout.rhs
+Source3: zprofile.rhs
+Source4: zshrc.rhs
+Source5: zshenv.rhs
+Source6: dotzshrc
+
+Patch1:  create-file.patch
+
+BuildRequires: autoconf
+BuildRequires: coreutils
+BuildRequires: gawk
+BuildRequires: gcc
+BuildRequires: gdbm-devel
+BuildRequires: libcap-devel
+BuildRequires: ncurses-devel
+BuildRequires: pcre-devel
+BuildRequires: sed
+BuildRequires: texi2html
+BuildRequires: texinfo
+Requires(post): info grep
+Requires(preun): info
+Requires(postun): coreutils grep
+
+# the hostname package is not available on RHEL-6
+%if 12 < 0%{?fedora} || 6 < 0%{?rhel}
+BuildRequires: hostname
+%else
+# /bin and /usr/bin are separate directories on RHEL-6
+%define _bindir /bin
+%endif
+
+Provides: /bin/zsh
+
+%description
+Spec file for zsh to test lookaside cache
+
+%package html
+Summary: Zsh shell manual in html format
+BuildArch:	noarch
+
+%description html
+Spec file for zsh to test lookaside cache
+
+%prep
+%autosetup -p1
+
+# enable parallel build
+sed -e 's|^\.NOTPARALLEL|#.NOTPARALLEL|' -i 'Config/defs.mk.in'
+
+%build
+# make build of run-time loadable modules work again (#1535422)
+%undefine _strict_symbol_defs_build
+
+# make loading of module's dependencies work again (#1277996)
+export LIBLDFLAGS='-z lazy'
+
+%configure \
+    --enable-etcdir=%{_sysconfdir} \
+    --with-tcsetpgrp \
+    --enable-maildir-support \
+    --enable-pcre
+
+# prevent the build from failing while running in parallel
+make -C Src headers
+make -C Src -f Makemod zsh{path,xmod}s.h version.h
+
+make %{?_smp_mflags} all html
+
+%check
+# Run the testsuite
+make check
+
+%install
+%make_install install.info \
+  fndir=%{_datadir}/%{name}/%{version}/functions \
+  sitefndir=%{_datadir}/%{name}/site-functions \
+  scriptdir=%{_datadir}/%{name}/%{version}/scripts \
+  sitescriptdir=%{_datadir}/%{name}/scripts \
+  runhelpdir=%{_datadir}/%{name}/%{version}/help
+
+rm -f $RPM_BUILD_ROOT%{_bindir}/zsh-%{version}
+rm -f $RPM_BUILD_ROOT%{_infodir}/dir
+
+mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}
+for i in %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5}; do
+    install -m 644 $i $RPM_BUILD_ROOT%{_sysconfdir}/"$(basename $i .rhs)"
+done
+
+mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/skel
+install -m 644 %{SOURCE6} $RPM_BUILD_ROOT%{_sysconfdir}/skel/.zshrc
+
+# This is just here to shut up rpmlint, and is very annoying.
+# Note that we can't chmod everything as then rpmlint will complain about
+# those without a she-bang line.
+for i in checkmail harden run-help zcalc zkbd; do
+    sed -i -e 's!/usr/local/bin/zsh!%{_bindir}/zsh!' \
+    $RPM_BUILD_ROOT%{_datadir}/zsh/%{version}/functions/$i
+    chmod +x $RPM_BUILD_ROOT%{_datadir}/zsh/%{version}/functions/$i
+done
+
+
+%post
+if [ "$1" = 1 ]; then
+  if [ ! -f %{_sysconfdir}/shells ] ; then
+    echo "%{_bindir}/%{name}" > %{_sysconfdir}/shells
+    echo "/bin/%{name}" >> %{_sysconfdir}/shells
+  else
+    grep -q "^%{_bindir}/%{name}$" %{_sysconfdir}/shells || echo "%{_bindir}/%{name}" >> %{_sysconfdir}/shells
+    grep -q "^/bin/%{name}$" %{_sysconfdir}/shells || echo "/bin/%{name}" >> %{_sysconfdir}/shells
+  fi
+fi
+
+if [ -f %{_infodir}/zsh.info.gz ]; then
+# This is needed so that --excludedocs works.
+/sbin/install-info %{_infodir}/zsh.info.gz %{_infodir}/dir \
+  --entry="* zsh: (zsh).			An enhanced bourne shell."
+fi
+
+
+%preun
+if [ "$1" = 0 ] ; then
+    if [ -f %{_infodir}/zsh.info.gz ]; then
+    # This is needed so that --excludedocs works.
+    /sbin/install-info --delete %{_infodir}/zsh.info.gz %{_infodir}/dir \
+      --entry="* zsh: (zsh).			An enhanced bourne shell."
+    fi
+fi
+
+%postun
+if [ "$1" = 0 ] && [ -f %{_sysconfdir}/shells ] ; then
+  sed -i '\!^%{_bindir}/%{name}$!d' %{_sysconfdir}/shells
+  sed -i '\!^/bin/%{name}$!d' %{_sysconfdir}/shells
+fi
+
+
+%files
+%doc README LICENCE Etc/BUGS Etc/CONTRIBUTORS Etc/FAQ FEATURES MACHINES
+%doc NEWS Etc/zsh-development-guide Etc/completion-style-guide
+%attr(755,root,root) %{_bindir}/zsh
+%{_mandir}/*/*
+%{_infodir}/*
+%{_datadir}/zsh
+%{_libdir}/zsh
+%config(noreplace) %{_sysconfdir}/skel/.z*
+%config(noreplace) %{_sysconfdir}/z*
+
+%files html
+%doc Doc/*.html
+
+%changelog
+* Fri Nov 30 2018 Kamil Dudka <kdudka@redhat.com> - 5.6.2-3
+- return non-zero exit status on nested parse error (#1654989)
diff --git a/tests/tests.yml b/tests/tests.yml
index 6e1486b..a0ee4b8 100644
--- a/tests/tests.yml
+++ b/tests/tests.yml
@@ -3,3 +3,4 @@
 - import_playbook: basic.yml
 - import_playbook: beakerlib.yml
 - import_playbook: avocado.yml
+- import_playbook: source.yml