From 479e0a39574b0efebd18294f60b43be177f5acd5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 07 2019 14:41:35 +0000 Subject: Provide a way to run the tests in containers using RPMs on F29 and C7 Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/containers/centos7-rpms b/dev/containers/centos7-rpms deleted file mode 100644 index 8ed9083..0000000 --- a/dev/containers/centos7-rpms +++ /dev/null @@ -1,40 +0,0 @@ -FROM centos:7 - -RUN yum -y install \ - epel-release - -RUN yum -y install \ - yum-utils \ - rpm-build \ - python-setuptools \ - python-coverage \ - python-nose \ - python-mock \ - python-docutils \ - python-enum \ - python-flask \ - python2-flake8 \ - redis \ - which - -RUN cd /pagure && \ - yum install --enablerepo=epel-testing -y \ - `grep "Requires:" /pagure/files/pagure.spec | \ - awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \ - sed -e "s|%{python_pkgversion}||"` && \ - yum clean all && \ - localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN \ - sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \ - sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \ - sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \ - /pagure/requirements.txt && \ - cd /pagure && python setup.py build - - -#ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 -WORKDIR /pagure -ENTRYPOINT ["python2", "/pagure/runtests.py", "run", "-f", "--py2"] -#ENTRYPOINT ["py.test", "-x", "tests/"] -CMD [] diff --git a/dev/containers/centos7-rpms-py2 b/dev/containers/centos7-rpms-py2 new file mode 100644 index 0000000..26ee266 --- /dev/null +++ b/dev/containers/centos7-rpms-py2 @@ -0,0 +1,53 @@ +FROM centos:7 + +RUN yum -y install \ + epel-release + +ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo + +RUN yum -y install \ + yum-utils \ + rpm-build \ + python-setuptools \ + python-coverage \ + python-nose \ + python-mock \ + python-docutils \ + python-enum \ + python-flask \ + python-flake8 \ + redis \ + which \ + git \ + repoSpanner \ + repoSpanner-bridge + +RUN yum install -y https://kojipkgs.fedoraproject.org//work/tasks/5657/32535657/python-werkzeug-0.9.6-1.el7.noarch.rpm + +RUN cd / \ + && git clone https://pagure.io/pagure.git \ + && chmod +x /pagure/dev/containers/runtests_py2.sh + +# Install all the requirements from the spec file and replace the macro +# %{python_pkgversion} by empty string which thus installs all the py2 +# version of the dependencies. +RUN cd /pagure && \ + yum install --enablerepo=epel-testing -y \ + `rpmspec -q --requires /pagure/files/pagure.spec | \ + sed -e "s|%{python_pkgversion}||"` && \ + yum clean all && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 + +# The old version of setuptools in CentOS7 does not support restrictions on +# the requirements file, so drop them +RUN \ + sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \ + sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \ + sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \ + /pagure/requirements.txt && \ + cd /pagure && python setup.py build + + +WORKDIR /pagure +ENTRYPOINT ["/pagure/dev/containers/runtests_py2.sh"] +CMD [] diff --git a/dev/containers/f29-rpms-py3 b/dev/containers/f29-rpms-py3 new file mode 100644 index 0000000..bb000a7 --- /dev/null +++ b/dev/containers/f29-rpms-py3 @@ -0,0 +1,31 @@ +FROM fedora:29 + +RUN dnf -y install \ + python3-setuptools \ + python3-coverage \ + python3-nose \ + python3-mock \ + python3-docutils \ + python3-black \ + python3-flake8 \ + redis \ + which \ + git + +RUN cd / \ + && git clone https://pagure.io/pagure.git \ + && chmod +x /pagure/dev/containers/runtests_py3.sh + +# Install all the requirements from the spec file and replace the macro +# %{python_pkgversion} by '3' which thus installs all the py3 version of +# the dependencies. +RUN dnf install -y `grep "Requires:" /pagure/files/pagure.spec | \ + awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \ + sed -e "s|%{python_pkgversion}|3|"` && \ + dnf clean all && \ + cd /pagure && python setup.py build + + +WORKDIR /pagure +ENTRYPOINT ["/pagure/dev/containers/runtests_py3.sh"] +CMD [] diff --git a/dev/containers/runtests_py2.sh b/dev/containers/runtests_py2.sh new file mode 100644 index 0000000..b161b17 --- /dev/null +++ b/dev/containers/runtests_py2.sh @@ -0,0 +1,25 @@ +#!/bin/bash + + +ls -l / + +echo "============== ENVIRONMENT =============" +/usr/bin/env +echo "============== END ENVIRONMENT =============" + +if [ -n "$REPO" -a -n "$BRANCH" ]; then +git remote rm proposed || true +git gc --auto +git remote add proposed "$REPO" +git fetch proposed +git checkout origin/master +git config --global user.email "you@example.com" +git config --global user.name "Your Name" +git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + +echo "Running tests for branch $BRANCH of repo $REPO" +echo "Last commits:" +git log -2 +fi + +python2 /pagure/runtests.py run -f --py2 diff --git a/dev/containers/runtests_py3.sh b/dev/containers/runtests_py3.sh new file mode 100644 index 0000000..b36a14a --- /dev/null +++ b/dev/containers/runtests_py3.sh @@ -0,0 +1,25 @@ +#!/bin/bash + + +ls -l / + +echo "============== ENVIRONMENT =============" +/usr/bin/env +echo "============== END ENVIRONMENT =============" + +if [ -n "$REPO" -a -n "$BRANCH" ]; then +git remote rm proposed || true +git gc --auto +git remote add proposed "$REPO" +git fetch proposed +git checkout origin/master +git config --global user.email "you@example.com" +git config --global user.name "Your Name" +git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + +echo "Running tests for branch $BRANCH of repo $REPO" +echo "Last commits:" +git log -2 +fi + +python3 /pagure/runtests.py run -f --py3 diff --git a/dev/containers/test_env_template b/dev/containers/test_env_template deleted file mode 100644 index dc5869b..0000000 --- a/dev/containers/test_env_template +++ /dev/null @@ -1,15 +0,0 @@ -FROM $base_image - -$epel_pkg -$infra_repo - -RUN $pkg_mgr install -y $pkg_list \ - && $pkg_mgr clean all \ -# for the hooks we need to have /usr/bin/python - && cp /usr/bin/python3 /usr/bin/python \ -# for the alembic test we need /usr/bin/alembic - && cp /usr/bin/alembic-3 /usr/bin/alembic - -WORKDIR /pagure -ENTRYPOINT [ "/pagure/runtests.py", "run" ] -CMD [] diff --git a/dev/run-tests-container.py b/dev/run-tests-container.py old mode 100755 new mode 100644 index 1f44b4c..73fe99a --- a/dev/run-tests-container.py +++ b/dev/run-tests-container.py @@ -1,21 +1,8 @@ -#! /usr/bin/python3 +#! /usr/bin/env python + import argparse import os import subprocess as sp -from string import Template - -TEMPLATE = "dev/docker/test_env_template" - -PKG_LIST = "python3-alembic python3-arrow python3-binaryornot \ - python3-bleach python3-blinker python3-chardet python3-cryptography \ - python3-docutils python3-flask python3-fedora-flask \ - python3-flask-wtf python3-bcrypt python3-jinja2 \ - python3-markdown python3-munch python3-openid-cla \ - python3-openid-teams python3-psutil python3-pygit2 python3-pillow \ - python3-sqlalchemy python3-straight-plugin python3-wtforms \ - python3-nose python3-coverage python3-mock python3-mock \ - python3-eventlet python3-flask-oidc python3-flake8 python3-celery \ - python3-redis python3-trololio python3-beautifulsoup4 python3-black redis vim git" def setup_parser(): @@ -56,69 +43,38 @@ if __name__ == "__main__": args = parser.parse_args() if args.centos is True: - base_image = "centos:7" - pkg_mgr = "yum" - epel_pkg = "RUN yum -y install epel-release" - infra_repo = ( - "ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo" - ) - container_name = "pagure-test-centos" - PKG_LIST += "python34 python34-coverage" + container_names = ["pagure-c7-rpms-py2"] + container_files = ["centos7-rpms-py2"] + elif args.fedora is True: + container_names = ["pagure-f29-rpms-py3"] + container_files = ["f29-rpms-py3"] else: - base_image = "registry.fedoraproject.org/fedora:latest" - pkg_mgr = "dnf" - container_name = "pagure-test-fedora" - epel_pkg = "" - infra_repo = "" + container_names = ["pagure-f29-rpms-py3", "pagure-c7-rpms-py2"] + container_files = ["f29-rpms-py3", "centos7-rpms-py2"] - with open(TEMPLATE, "r") as fp: - t = Template(fp.read()) - with open("dev/docker/test_env", "w") as fp: - fp.write( - t.substitute( - base_image=base_image, - pkg_list=PKG_LIST, - pkg_mgr=pkg_mgr, - epel_pkg=epel_pkg, - infra_repo=infra_repo, + for idx, container_name in enumerate(container_names): + if args.skip_build is not False: + print("------ Building Container Image -----") + sp.call( + [ + "podman", + "build", + "--rm", + "-t", + container_name, + "-f", + "dev/containers/%s" % container_files[idx], + "dev/containers", + ] ) - ) - - if args.skip_build is not False: - print("------ Building Docker Image -----") - sp.run( - [ - "podman", - "build", - "--rm", - "-t", - container_name, - "-f", - "dev/docker/test_env", - "dev/docker", - ] - ) - if args.shell: - print("--------- Shelling in the container --------------") - command = [ - "podman", - "run", - "-it", - "--rm", - "--name", - container_name, - "-v", - "{}:/pagure".format(os.getcwd()), - "--entrypoint=/bin/bash", - container_name, - ] - sp.run(command) - else: + result_path = "{}/results_{}".format(os.getcwd(), container_files[idx]) + if not os.path.exists(result_path): + os.mkdir(result_path) - print("--------- Running Test --------------") - sp.run( - [ + if args.shell: + print("--------- Shelling in the container --------------") + command = [ "podman", "run", "-it", @@ -126,8 +82,34 @@ if __name__ == "__main__": "--name", container_name, "-v", - "{}:/pagure".format(os.getcwd()), + "{}/results_{}:/pagure/results".format( + os.getcwd(), container_files[idx]), + "-e", + "BRANCH=$BRANCH", + "-e", + "REPO=$REPO", + "--entrypoint=/bin/bash", container_name, - args.test_case, ] - ) + sp.call(command) + else: + print("--------- Running Test --------------") + sp.call( + [ + "podman", + "run", + "-it", + "--rm", + "--name", + container_name, + "-v", + "{}/results_{}:/pagure/results".format( + os.getcwd(), container_files[idx]), + "-e", + "BRANCH={}".format(os.environ.get("BRANCH") or ""), + "-e", + "REPO={}".format(os.environ.get("REPO") or ""), + container_name, + args.test_case, + ] + ) diff --git a/run_ci_tests_containers.sh b/run_ci_tests_containers.sh new file mode 100644 index 0000000..38e3d14 --- /dev/null +++ b/run_ci_tests_containers.sh @@ -0,0 +1,38 @@ +set -x + +yum install -y podman + +sysctl -w fs.file-max=2048 + +set -e + +echo $BRANCH $REPO + +podman build --rm -t pagure-f29-rpms-py3 \ + -f dev/containers/f29-rpms-py3 \ + dev/containers + +if [ ! -d `pwd`/results_f29-rpms-py3 ]; then + mkdir `pwd`/results_f29-rpms-py3; +fi + +podman run --rm -it --name pagure-f29-rpms-py3 \ + -v `pwd`/results_f29-rpms-py3:/pagure/results \ + -e BRANCH=$BRANCH \ + -e REPO=$REPO \ + pagure-f29-rpms-py3 + + +podman build --rm -t pagure-c7-rpms-py2 \ + -f dev/containers/centos7-rpms-py2 \ + dev/containers + +if [ ! -d `pwd`/results_centos7-rpms-py2 ]; then + mkdir `pwd`/results_centos7-rpms-py2; +fi + +podman run --rm -it --name pagure-c7-rpms-py2 \ + -v `pwd`/results_centos7-rpms-py2:/pagure/results \ + -e BRANCH=$BRANCH \ + -e REPO=$REPO \ + pagure-c7-rpms-py2