From d548a2dd38f0ed46621fd8897f62f243b6d57b4a Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 24 2017 21:14:40 +0000 Subject: [PATCH 1/3] Add docker and rhts roles. --- diff --git a/roles/standard-test-docker/README.md b/roles/standard-test-docker/README.md new file mode 100644 index 0000000..c7b3c22 --- /dev/null +++ b/roles/standard-test-docker/README.md @@ -0,0 +1,11 @@ +# Ansible role for docker image subjects + +Put this role in your test_docker.yml playbook. You'll need +to have the following variables defined: + + * subjects: A docker image name + * artifacts: An artifacts directory + * playbooks: A playbook to run inside of the container + +Set the FEDORA_TEST_DIAGNOSE=1 environment variable to diagnose +any issues in the docker container. diff --git a/roles/standard-test-docker/defaults/main.yml b/roles/standard-test-docker/defaults/main.yml new file mode 100644 index 0000000..6c8e928 --- /dev/null +++ b/roles/standard-test-docker/defaults/main.yml @@ -0,0 +1,2 @@ +--- +docker_python_version: 2 diff --git a/roles/standard-test-docker/files/docker-inventory b/roles/standard-test-docker/files/docker-inventory new file mode 100644 index 0000000..3a5fcf6 --- /dev/null +++ b/roles/standard-test-docker/files/docker-inventory @@ -0,0 +1 @@ +localhost ansible_ssh_port=2222 ansible_ssh_host=127.0.0.3 ansible_ssh_user=root ansible_ssh_pass=foobar ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' diff --git a/roles/standard-test-docker/files/docker-run-ssh b/roles/standard-test-docker/files/docker-run-ssh new file mode 100644 index 0000000..16faf06 --- /dev/null +++ b/roles/standard-test-docker/files/docker-run-ssh @@ -0,0 +1,16 @@ +#!/bin/sh +python_version=2 +while [ $# -gt 0 ]; do + case "$1" in + --py3) python_version=3 ;; + esac + shift +done +dnf install -y openssh-server python${python_version}-dnf +ssh-keygen -q -t rsa -N '' -f /etc/ssh/ssh_host_rsa_key +echo 'root:foobar' | chpasswd +# start sshd in background +/usr/sbin/sshd +echo SSHD READY +# wait forever +tail -f /dev/null diff --git a/roles/standard-test-docker/tasks/main.yml b/roles/standard-test-docker/tasks/main.yml new file mode 100644 index 0000000..a2830a5 --- /dev/null +++ b/roles/standard-test-docker/tasks/main.yml @@ -0,0 +1,81 @@ +--- +- name: Install the docker requirements + package: name="{{item}}" state=latest + with_items: + - docker + +- name: Make artifacts directory + file: path={{ artifacts }} state=directory owner=root mode=755 recurse=yes + +- name: Start the docker service + service: + name: docker + state: running + +- name: Initialize docker and ansible playbook extra args variables + set_fact: + my_docker_extra_args: null + my_playbook_extra_args: null + +- name: Configure python version-dependent argument settings + set_fact: + my_playbook_extra_args: "{{my_playbook_extra_args}} -e ansible_python_interpreter='/usr/bin/python{{docker_python_version}}'" + my_docker_extra_args: "{{my_docker_extra_args}} --py{{docker_python_version}}" + when: docker_python_version is defined and docker_python_version != 2 + +- name: Special case - pass along 'rpms' variable to playbook + set_fact: + # note the quoting here: ansible needs to see the single quotes + my_playbook_extra_args: "{{my_playbook_extra_args}} -e \"rpms='{{rpms}}'\"" + when: rpms|default("") != "" + +- name: Start ansible-ready docker container running SSHD + shell: > + docker run -d + -p '{{docker_ssh_port}}:22' + -v '{{artifacts}}:/artifacts:z,rw' + -v '{{role_path}}/files/docker-run-ssh:/run.sh:z' + '{{subjects}}' + /bin/sh -ex /run.sh {{my_docker_extra_args}} + register: docker_run_output + +- name: Capture docker container ID + set_fact: container_id="{{docker_run_output.stdout}}" + +- block: + - name: Wait for container to initialize + wait_for: port="{{docker_ssh_port}}" search_regex=OpenSSH + +# ******************************** +# HELP! +# There must be a better way to run the actual test playbook without running +# another instance of ansible-playbook in an external shell command. +# The external run means we can't track the return status of the invidual +# tasks in the test playbook--which is the whole point. +# ******************************** + - name: Run the playbook in the container + shell: > + ANSIBLE_LOG_PATH='{{artifacts}}/container-playbook.log' + ansible-playbook '{{playbooks}}' + -i '{{role_path}}/files/docker-inventory' + -e artifacts='/artifacts' + {{my_playbook_extra_args}} + register: playbook_output + + - debug: var=playbook_output + + always: + - name: Pause if diagnosing container + pause: + prompt: |- + To diagnose the container, you can use one of the following commands: + ssh -p {{docker_ssh_port}} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@127.0.0.3 # password: foobar + docker exec -it {{ container_id }} /bin/bash + Continue when ready + when: lookup('env','FEDORA_TEST_DIAGNOSE')|bool + + - name: Save the container log as an artifact + shell: docker logs "{{ container_id }}" >"{{artifacts}}/docker.log" 2>&1 + + - name: Clean up the docker container + shell: docker rm -f "{{ container_id }}" diff --git a/roles/standard-test-docker/vars/main.yml b/roles/standard-test-docker/vars/main.yml new file mode 100644 index 0000000..243bb7e --- /dev/null +++ b/roles/standard-test-docker/vars/main.yml @@ -0,0 +1,5 @@ +--- +artifacts: ./artifacts +log: "{{artifacts}}/docker.log" +subjects: [] +docker_ssh_port: 2222 diff --git a/roles/standard-test-rhts/README.md b/roles/standard-test-rhts/README.md new file mode 100644 index 0000000..50efb62 --- /dev/null +++ b/roles/standard-test-rhts/README.md @@ -0,0 +1,9 @@ +# Ansible role for RHTS (Red Hat Test Suite) tests + +Put this role in your test_local.yml playbook. You'll need +to have the following variables defined: + + * tests: A list of RHTS test directories + * artifacts: An artifacts directory + * required_packages: A list of prerequisite packages required by RHTS tests + * rpms: Space separated list of RPMs to install (optional, may include SRPMs) diff --git a/roles/standard-test-rhts/defaults/main.yml b/roles/standard-test-rhts/defaults/main.yml new file mode 100644 index 0000000..be42113 --- /dev/null +++ b/roles/standard-test-rhts/defaults/main.yml @@ -0,0 +1,2 @@ +--- +rpms: [] diff --git a/roles/standard-test-rhts/files/job2text.xsl b/roles/standard-test-rhts/files/job2text.xsl new file mode 100644 index 0000000..c8eb693 --- /dev/null +++ b/roles/standard-test-rhts/files/job2text.xsl @@ -0,0 +1,14 @@ + + + + + + + + + + + + diff --git a/roles/standard-test-rhts/tasks/main.yml b/roles/standard-test-rhts/tasks/main.yml new file mode 100644 index 0000000..f579630 --- /dev/null +++ b/roles/standard-test-rhts/tasks/main.yml @@ -0,0 +1,188 @@ +--- +- name: Install the RHTS pre-requirements + package: name={{item}} state=latest + with_items: + - dnf-plugins-core # COPR plugin needed + - beakerlib + - make + - createrepo + +- name: Enable COPR repo for restraint + shell: dnf copr -y enable bpeck/restraint + +- name: Install restraint from COPR repo + package: name={{item}} state=latest + with_items: + - restraint + - restraint-rhts + - restraint-client + +- name: Install any test-specific package requirements + package: name={{item}} state=latest + with_items: + - "{{ required_packages }}" + +- name: Create legacy beakerlib directories + file: + dest: "{{ item }}" + state: directory + with_items: + - /usr/lib/beakerlib + - /usr/share/rhts-library + +- name: Create legacy beakerlib links + file: + src: /usr/share/beakerlib/beakerlib.sh + dest: "{{ item }}" + state: link + with_items: + - /usr/lib/beakerlib/beakerlib.sh + - /usr/share/rhts-library/rhtslib.sh + +- name: Create RPM repo directory + file: + dest: "{{ repo_dir }}" + state: directory + +- name: Initialize RPM list + set_fact: + my_rpm_list: [] + +- name: Split 'rpms' argument into a list + set_fact: + my_rpm_list: "{{ rpms.split(' ') | unique }}" + when: rpms is defined and rpms|length > 0 + +- name: Copy RPMs to target + copy: + src: "{{ item }}" + dest: "{{ repo_dir }}/" + with_items: + - "{{ my_rpm_list }}" + +- name: Generate RPM repo metadata + shell: createrepo "{{ repo_dir }}" + +- name: Add RPM repo + yum_repository: + name: local + description: local repo + baseurl: "file://{{ repo_dir }}" + gpgcheck: no + +- name: Set list of SRPMs + set_fact: + my_srpm_list: "{{ my_rpm_list + | map('regex_search', '^.*\\.src\\.rpm$') + | select('string') | list }}" + +- name: Set list of non-SRPMs + set_fact: + my_nonsrpm_list: "{{ my_rpm_list | difference(my_srpm_list) }}" + +- name: Map non-SRPMs to target paths to be installed + set_fact: + my_install_rpm_list: "{{ my_nonsrpm_list | map('basename') + | map('regex_replace', '^', repo_dir ~ '/') + | list }}" + +- name: Install RPMs + shell: dnf install -y {{ my_install_rpm_list | join(' ') }} + when: my_install_rpm_list|length > 0 + +- name: Create tests directory + file: + dest: "{{ test_dir }}" + state: directory + +- name: Copy tests to target + copy: + src: "{{ playbook_dir }}/{{ item }}" + dest: "{{ test_dir }}/" + with_items: + - "{{ tests }}" + +- name: Generate test archives for restraint + shell: tar czf "{{ test_dir }}/{{ item | basename }}.tgz" -C "{{ test_dir }}" "{{ item | basename }}" + with_items: + - "{{ tests }}" + +- name: Write restraint job XML file intro + shell: | + echo "" >"{{ job_xml_file }}" + echo " " >>"{{ job_xml_file }}" + echo " " >>"{{ job_xml_file }}" + +- name: Write restraint job XML file recipes + shell: | + echo " " >>"{{ job_xml_file }}" + echo " " >>"{{ job_xml_file }}" + echo " " >>"{{ job_xml_file }}" + with_items: + - "{{ tests }}" + +- name: Write restraint job XML file outro + shell: | + echo " " >>"{{ job_xml_file }}" + echo " " >>"{{ job_xml_file }}" + echo "" >>"{{ job_xml_file }}" + +- name: Copy restraint results conversion stylesheet to target + copy: + src: job2text.xsl + dest: /usr/local/share/ + +- name: Create root SSH directory + file: + dest: /root/.ssh + state: directory + +- name: Create root SSH key pair + shell: ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa + args: + creates: /root/.ssh/id_rsa + +- name: Configure password-less local SSH for restraint + shell: | + cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys + chmod 644 /root/.ssh/authorized_keys + +- name: Make artifacts directory + file: path={{ artifacts }} state=directory owner=root mode=755 recurse=yes + +- name: Start restraintd + shell: nohup /usr/bin/restraintd >"{{ artifacts }}/restraintd.log" 2>&1 & + +- name: Start local web server for restraint + shell: nohup /usr/bin/python -m SimpleHTTPServer "{{ local_www_port }}" >"{{ artifacts }}/httpd.log" 2>&1 & + args: + chdir: "{{ local_www_dir }}/" + +- block: + - name: Execute RHTS tests using restraint + shell: /usr/bin/restraint --host localhost --job "{{ job_xml_file }}" >"{{ artifacts }}/restraint.log" 2>&1 + args: + chdir: "{{ artifacts }}/" + ignore_errors: True + + - name: Extract job output directory from restraint logfile + shell: sed -n 's/^Using \([^ ]*\).*$/\1/p' "{{ artifacts }}/restraint.log" + register: restraint_job_dir + + - name: Set name of restraint XML job results file + set_fact: results_xml="{{ artifacts }}/{{restraint_job_dir.stdout}}/job.xml" + + - name: Convert restraint XML job results to text as main output artifact + shell: xsltproc /usr/local/share/job2text.xsl "{{ results_xml }}" >"{{ artifacts }}/test.log" + + - name: Check the results for failures + shell: grep '^FAIL ' "{{ artifacts }}/test.log" + register: test_fails + failed_when: test_fails.stdout or test_fails.stderr + +- always: + - name: Pull out the logs + fetch: + dest: "{{artifacts}}/" + src: "{{artifacts}}/" + flat: yes diff --git a/roles/standard-test-rhts/vars/main.yml b/roles/standard-test-rhts/vars/main.yml new file mode 100644 index 0000000..9dbdcfa --- /dev/null +++ b/roles/standard-test-rhts/vars/main.yml @@ -0,0 +1,9 @@ +--- +artifacts: ./artifacts +tests: [] +required_packages: [] +local_www_dir: "/var/www-test" +local_www_port: 8888 +test_dir: "{{local_www_dir}}" +repo_dir: "/var/tmp/local_repo" +job_xml_file: "{{local_www_dir}}/job.xml" From 654dd2aa826287b12d7368bb230ceb0a72012ffc Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 26 2017 14:17:49 +0000 Subject: [PATCH 2/3] Add task to rhts role to make job ouput directory tree readable by all. --- diff --git a/roles/standard-test-rhts/tasks/main.yml b/roles/standard-test-rhts/tasks/main.yml index f579630..2f762eb 100644 --- a/roles/standard-test-rhts/tasks/main.yml +++ b/roles/standard-test-rhts/tasks/main.yml @@ -169,6 +169,12 @@ shell: sed -n 's/^Using \([^ ]*\).*$/\1/p' "{{ artifacts }}/restraint.log" register: restraint_job_dir + - name: Make job ouput directory tree readable by all + file: + path: "{{ artifacts }}/{{restraint_job_dir.stdout}}" + mode: u=rwX,g=rX,o=rX + recurse: yes + - name: Set name of restraint XML job results file set_fact: results_xml="{{ artifacts }}/{{restraint_job_dir.stdout}}/job.xml" From de1384b2e659b0415556ccfd8d44ef2c54d47a71 Mon Sep 17 00:00:00 2001 From: Merlin Mathesius Date: May 31 2017 21:50:28 +0000 Subject: [PATCH 3/3] Use template module to generate restraint job.xml file. Add comment explaining why service module is not used to start restraintd. --- diff --git a/roles/standard-test-rhts/tasks/main.yml b/roles/standard-test-rhts/tasks/main.yml index 2f762eb..89cc28c 100644 --- a/roles/standard-test-rhts/tasks/main.yml +++ b/roles/standard-test-rhts/tasks/main.yml @@ -107,25 +107,11 @@ with_items: - "{{ tests }}" -- name: Write restraint job XML file intro - shell: | - echo "" >"{{ job_xml_file }}" - echo " " >>"{{ job_xml_file }}" - echo " " >>"{{ job_xml_file }}" - -- name: Write restraint job XML file recipes - shell: | - echo " " >>"{{ job_xml_file }}" - echo " " >>"{{ job_xml_file }}" - echo " " >>"{{ job_xml_file }}" - with_items: - - "{{ tests }}" - -- name: Write restraint job XML file outro - shell: | - echo " " >>"{{ job_xml_file }}" - echo " " >>"{{ job_xml_file }}" - echo "" >>"{{ job_xml_file }}" +- name: Generate restraint job XML file + template: + # template references variables 'tests' and 'local_www_port' + src: job.xml.j2 + dest: "{{ job_xml_file }}" - name: Copy restraint results conversion stylesheet to target copy: @@ -151,6 +137,8 @@ file: path={{ artifacts }} state=directory owner=root mode=755 recurse=yes - name: Start restraintd + # instead of using service module, daemon is started directly since we want the + # output as an artifact (and this could be running in a container) shell: nohup /usr/bin/restraintd >"{{ artifacts }}/restraintd.log" 2>&1 & - name: Start local web server for restraint diff --git a/roles/standard-test-rhts/templates/job.xml.j2 b/roles/standard-test-rhts/templates/job.xml.j2 new file mode 100644 index 0000000..c490d43 --- /dev/null +++ b/roles/standard-test-rhts/templates/job.xml.j2 @@ -0,0 +1,11 @@ + + + +{% for item in tests %} + + + +{% endfor %} + + +