From e917e1b40cec8c892a173e705e744fe1011503e3 Mon Sep 17 00:00:00 2001 From: Stef Walter Date: Nov 11 2018 10:18:44 +0000 Subject: [PATCH 1/3] roles: Fix broken standard-test-avocado role This always line needs to be yaml'd correctly, otherwise we get the following failure from Ansible ERROR! 'always' keyword cannot be used without 'block' --- diff --git a/roles/standard-test-avocado/tasks/main.yml b/roles/standard-test-avocado/tasks/main.yml index 22d720e..341ace2 100644 --- a/roles/standard-test-avocado/tasks/main.yml +++ b/roles/standard-test-avocado/tasks/main.yml @@ -12,7 +12,7 @@ - name: Execute the avocado test shell: exec 2>>{{ remote_artifacts }}/test.log 1>>{{ remote_artifacts }}/test.log; MODULE=rpm python -m avocado run --job-results-dir {{ remote_artifacts }}/ {{tests|join(' ') }} -- always: + always: - name: Pull out the logs synchronize: dest: "{{ artifacts }}/" From 235f8c0beb0126517639e361279bf77f9444f109 Mon Sep 17 00:00:00 2001 From: Stef Walter Date: Nov 11 2018 10:18:44 +0000 Subject: [PATCH 2/3] scripts: merge-standard-inventory ignore non-existant directory The merge-standard-inventory script should ignore a non-existant directory. This bug was caught by the integration tests. Otherwise we see this problem: Traceback (most recent call last): File "/data/src/standard-test-roles/scripts/merge-standard-inventory", line 208, in sys.exit(main(sys.argv)) File "/data/src/standard-test-roles/scripts/merge-standard-inventory", line 37, in main merged_data = merge_standard_inventories(argv[1:]) File "/data/src/standard-test-roles/scripts/merge-standard-inventory", line 82, in merge_standard_inventories for i in os.listdir(inventory_dir): OSError: [Errno 2] No such file or directory: '/usr/share/ansible/inventory' --- diff --git a/scripts/merge-standard-inventory b/scripts/merge-standard-inventory index cfcc2bd..61f2af1 100755 --- a/scripts/merge-standard-inventory +++ b/scripts/merge-standard-inventory @@ -79,7 +79,7 @@ def merge_standard_inventories(args): merged = Inventory() - for i in os.listdir(inventory_dir): + for i in os.path.exists(inventory_dir) and os.listdir(inventory_dir) or []: ipath = os.path.join(inventory_dir, i) if not i.startswith("standard-inventory-"): continue From f472e0366f0bc491b6e669ae560698c8fc90e92f Mon Sep 17 00:00:00 2001 From: Stef Walter Date: Nov 11 2018 10:18:44 +0000 Subject: [PATCH 3/3] tests: Distribute and invoke our own self-tests in Jenkins When we open a pull request to this repo we should be testing our own tests. Lets move the tests here, and gate in our own upstream repo ... before we even get to an RPM. Obviously we can use these tests again when we get to dist-git. --- diff --git a/tests/avocado.yml b/tests/avocado.yml new file mode 100644 index 0000000..846cd8c --- /dev/null +++ b/tests/avocado.yml @@ -0,0 +1,24 @@ +# Tests for avocado role +- hosts: localhost + tags: + - classic + - container + roles: + - role: standard-test-avocado + tests: + - /bin/true + - role: standard-test-avocado + tests: + - /bin/false + ignore_errors: yes + tasks: + - name: Read test.log on test environment + set_fact: + test_log: "{{ lookup('file', artifacts + '/test.log') }}" + - name: Check for FAIL on test.log on test environment + fail: msg="Could not find expected text on test log" + when: > + (test_log.find('/bin/false: FAIL') == -1) or + (test_log.find('/bin/true: PASS') == -1) + # We should fail when we can not find FAIL string on log + # -1 means string not found diff --git a/tests/basic.yml b/tests/basic.yml new file mode 100644 index 0000000..8233a04 --- /dev/null +++ b/tests/basic.yml @@ -0,0 +1,38 @@ +- import_playbook: prepare.yml + +# Tests for basic role +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-basic + tests: + - test-basic-simple + - test-basic-parameters: + dir: ./ + run: echo "check parameters on basic role" | grep "check parameters on basic role" + required_packages: + # Test if we can install required packages + - "{{req_pkg}}" + tasks: + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml + +# Make sure the role behaves correctly if test fails +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-basic + tests: + - test-basic-fail + ignore_errors: yes + tasks: + # 'verify_failed_test' tasks should run after 'test-basic-fail' + - import_tasks: shared-tasks/verify_failed_test.yml + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml diff --git a/tests/beakerlib.yml b/tests/beakerlib.yml new file mode 100644 index 0000000..5f84bda --- /dev/null +++ b/tests/beakerlib.yml @@ -0,0 +1,35 @@ +- import_playbook: prepare.yml + +# Tests for beakerlib role +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-beakerlib + tests: + - test-beakerlib-simple + required_packages: + - "{{req_pkg}}" + tasks: + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml + + +# Make sure the role behaves correctly if test fails +- hosts: localhost + tags: + - atomic + - classic + - container + roles: + - role: standard-test-beakerlib + tests: + - test-beakerlib-fail + ignore_errors: yes + tasks: + # 'tests_verify_failed_test' tasks should run after 'test-beakerlib-fail' + - import_tasks: shared-tasks/verify_failed_test.yml + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml diff --git a/tests/inventory.yml b/tests/inventory.yml new file mode 100644 index 0000000..9cd725d --- /dev/null +++ b/tests/inventory.yml @@ -0,0 +1,14 @@ +- hosts: localhost + tags: + - always + tasks: + - name: Get inventory with empty input + command: merge-standard-inventory + environment: + # Reset vars to default values + TEST_SUBJECTS: + register: inventory + delegate_to: localhost + - name: Assert merge-standard-inventory produces correct JSON inventory (empty) + fail: + when: inventory.stdout != '{}' diff --git a/tests/invoke-integration.sh b/tests/invoke-integration.sh new file mode 100755 index 0000000..96c577b --- /dev/null +++ b/tests/invoke-integration.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# +# This invokes the tests using the local inventory and roles rather +# than what's installed on system. This is aslight deviation from +# what's specified in the standard test interface +# + +cd $(dirname $0)/.. + +rm -rf /tmp/artifacts/ /tmp/local-artifacts/ +TEST_ARTIFACTS=/tmp PATH=$PWD/scripts:$PATH ANSIBLE_INVENTORY=$PWD/inventory \ + ansible-playbook --extra-vars=artifacts=/tmp/local-artifacts --tags=classic tests/tests.yml diff --git a/tests/prepare.yml b/tests/prepare.yml new file mode 100644 index 0000000..4cc4cce --- /dev/null +++ b/tests/prepare.yml @@ -0,0 +1,13 @@ +# Preparation steps +- hosts: localhost + vars: + # Must be the same as in str-common/defaults/main.yml + artifacts: "{{ lookup('env', 'TEST_ARTIFACTS')|default('./artifacts', true) }}" + remote_artifacts: /tmp/artifacts/ + tasks: + - import_tasks: shared-tasks/artifacts.yml + tags: + - always + - import_tasks: shared-tasks/req-pkg.yml + tags: + - always diff --git a/tests/rhts.yml b/tests/rhts.yml new file mode 100644 index 0000000..d9b2ea8 --- /dev/null +++ b/tests/rhts.yml @@ -0,0 +1,33 @@ +- import_playbook: prepare.yml + +# Tests for rhts role only runs on classic tag +- hosts: localhost + tags: + - classic + roles: + - role: standard-test-rhts + tests: + - test-rhts-simple + required_packages: + # Test if we can install required packages + - "{{req_pkg}}" + tasks: + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml + +# Make sure the role behaves correctly if test fails +- hosts: localhost + tags: + - classic + roles: + - role: standard-test-rhts + tags: + - classic + tests: + - test-rhts-fail + ignore_errors: yes + tasks: + # 'verify_failed_test' tasks should run after 'testi-rhts-fail' + - import_tasks: shared-tasks/verify_failed_test.yml + - import_tasks: shared-tasks/artifacts_test_env.yml + - import_tasks: shared-tasks/artifacts_test_runner.yml diff --git a/tests/roles b/tests/roles new file mode 120000 index 0000000..d8c4472 --- /dev/null +++ b/tests/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/tests/shared-tasks/artifacts.yml b/tests/shared-tasks/artifacts.yml new file mode 100644 index 0000000..2c51da3 --- /dev/null +++ b/tests/shared-tasks/artifacts.yml @@ -0,0 +1,9 @@ +# STR requires that artifacts dir must be absent +- name: "Check if {{artifacts}} is absent at test-environment." + stat: + path: "{{remote_artifacts}}" + register: st_te + +- fail: + msg: "Remove {{remote_artifacts}} before test run at test-environment." + when: st_te.stat.exists diff --git a/tests/shared-tasks/artifacts_test_env.yml b/tests/shared-tasks/artifacts_test_env.yml new file mode 100644 index 0000000..4d8ae88 --- /dev/null +++ b/tests/shared-tasks/artifacts_test_env.yml @@ -0,0 +1,9 @@ +# These tasks should run after as test that fails +- name: "Check if {{remote_artifacts}} was created properly on test environment" + shell: "ls {{remote_artifacts}}/test.log" + +- name: "Clean up {{remote_artifacts}} to not affect other tests on test environment" + file: + state: absent + path: "{{remote_artifacts}}/" + diff --git a/tests/shared-tasks/artifacts_test_runner.yml b/tests/shared-tasks/artifacts_test_runner.yml new file mode 100644 index 0000000..85b59d8 --- /dev/null +++ b/tests/shared-tasks/artifacts_test_runner.yml @@ -0,0 +1,10 @@ +# These tasks should run after as test that fails +- name: "Check if {{artifacts}} was created properly on test runner" + shell: "ls {{artifacts}}/test.log" + delegate_to: localhost + +- name: "Clean up {{artifacts}} to not affect other tests on test runner" + file: + state: absent + path: "{{artifacts}}/" + delegate_to: localhost diff --git a/tests/shared-tasks/req-pkg.yml b/tests/shared-tasks/req-pkg.yml new file mode 100644 index 0000000..d871f41 --- /dev/null +++ b/tests/shared-tasks/req-pkg.yml @@ -0,0 +1,12 @@ +- import_role: + name: str-common-pkgs + tasks_from: inspect.yml + +- set_fact: + req_pkg: zsh + +- name: Remove a test package from test-environment + package: + name: "{{req_pkg}}" + state: absent + when: not is_atomic diff --git a/tests/shared-tasks/verify_failed_test.yml b/tests/shared-tasks/verify_failed_test.yml new file mode 100644 index 0000000..a51af94 --- /dev/null +++ b/tests/shared-tasks/verify_failed_test.yml @@ -0,0 +1,11 @@ +# These tasks should run after as test that fails +- name: Read test.log on test environment + shell: "cat {{remote_artifacts}}/test.log" + register: test_log + +- name: Check for FAIL on test.log on test environment + fail: msg="Could not find FAIL on test log" + when: test_log.stdout.find("FAIL") == -1 + # We should fail when we can not find FAIL string on log + # -1 means string not found + diff --git a/tests/test-basic-fail/runtest.sh b/tests/test-basic-fail/runtest.sh new file mode 100644 index 0000000..a7f2a1c --- /dev/null +++ b/tests/test-basic-fail/runtest.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Running a command that should fail" + +false + diff --git a/tests/test-basic-simple/runtest.sh b/tests/test-basic-simple/runtest.sh new file mode 100644 index 0000000..87c587b --- /dev/null +++ b/tests/test-basic-simple/runtest.sh @@ -0,0 +1,5 @@ +#!/bin/bash +PACKAGE="bash" + +echo "Using $PACKAGE version:" +rpm -q $PACKAGE diff --git a/tests/test-beakerlib-fail/runtest.sh b/tests/test-beakerlib-fail/runtest.sh new file mode 100644 index 0000000..f1e63f7 --- /dev/null +++ b/tests/test-beakerlib-fail/runtest.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +PACKAGE="bash" + +rlJournalStart + rlPhaseStartTest "Simple beakerlib test" + # Running a command that should fail + rlRun "false" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd diff --git a/tests/test-beakerlib-simple/runtest.sh b/tests/test-beakerlib-simple/runtest.sh new file mode 100644 index 0000000..fb77e47 --- /dev/null +++ b/tests/test-beakerlib-simple/runtest.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +PACKAGE="bash" + +rlJournalStart + rlPhaseStartTest "Simple beakerlib test" + rlRun "ls /" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd diff --git a/tests/test-rhts-fail/Makefile b/tests/test-rhts-fail/Makefile new file mode 100644 index 0000000..9ae494f --- /dev/null +++ b/tests/test-rhts-fail/Makefile @@ -0,0 +1,34 @@ +export TESTVERSION=1.0 + +BUILT_FILES= + +FILES=$(METADATA) runtest.sh Makefile + +.PHONY: all install clean + +run: $(FILES) build + ./runtest.sh + +build: $(BUILT_FILES) + test -x runtest.sh || chmod a+x runtest.sh + +clean: + rm -f *~ $(BUILT_FILES) + + +include /usr/share/rhts/lib/rhts-make.include + +$(METADATA): Makefile + @echo "Owner: Bruno Goncalves " > $(METADATA) + @echo "Name: $(TEST)" >> $(METADATA) + @echo "TestVersion: $(TESTVERSION)" >> $(METADATA) + @echo "Path: $(TEST_DIR)" >> $(METADATA) + @echo "Description: fail rhts test role" >> $(METADATA) + @echo "Type: Sanity" >> $(METADATA) + @echo "TestTime: 10m" >> $(METADATA) + @echo "Priority: Normal" >> $(METADATA) + @echo "License: GPLv2" >> $(METADATA) + @echo "Confidential: no" >> $(METADATA) + @echo "Destructive: no" >> $(METADATA) + + rhts-lint $(METADATA) diff --git a/tests/test-rhts-fail/runtest.sh b/tests/test-rhts-fail/runtest.sh new file mode 100644 index 0000000..f1e63f7 --- /dev/null +++ b/tests/test-rhts-fail/runtest.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +PACKAGE="bash" + +rlJournalStart + rlPhaseStartTest "Simple beakerlib test" + # Running a command that should fail + rlRun "false" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd diff --git a/tests/test-rhts-simple/Makefile b/tests/test-rhts-simple/Makefile new file mode 100644 index 0000000..eb0d996 --- /dev/null +++ b/tests/test-rhts-simple/Makefile @@ -0,0 +1,34 @@ +export TESTVERSION=1.0 + +BUILT_FILES= + +FILES=$(METADATA) runtest.sh Makefile + +.PHONY: all install clean + +run: $(FILES) build + ./runtest.sh + +build: $(BUILT_FILES) + test -x runtest.sh || chmod a+x runtest.sh + +clean: + rm -f *~ $(BUILT_FILES) + + +include /usr/share/rhts/lib/rhts-make.include + +$(METADATA): Makefile + @echo "Owner: Bruno Goncalves " > $(METADATA) + @echo "Name: $(TEST)" >> $(METADATA) + @echo "TestVersion: $(TESTVERSION)" >> $(METADATA) + @echo "Path: $(TEST_DIR)" >> $(METADATA) + @echo "Description: simple rhts test role" >> $(METADATA) + @echo "Type: Sanity" >> $(METADATA) + @echo "TestTime: 10m" >> $(METADATA) + @echo "Priority: Normal" >> $(METADATA) + @echo "License: GPLv2" >> $(METADATA) + @echo "Confidential: no" >> $(METADATA) + @echo "Destructive: no" >> $(METADATA) + + rhts-lint $(METADATA) diff --git a/tests/test-rhts-simple/runtest.sh b/tests/test-rhts-simple/runtest.sh new file mode 100644 index 0000000..fb77e47 --- /dev/null +++ b/tests/test-rhts-simple/runtest.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +PACKAGE="bash" + +rlJournalStart + rlPhaseStartTest "Simple beakerlib test" + rlRun "ls /" + rlPhaseEnd +rlJournalPrintText +rlJournalEnd diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..6e1486b --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,5 @@ +--- +- import_playbook: inventory.yml +- import_playbook: basic.yml +- import_playbook: beakerlib.yml +- import_playbook: avocado.yml