From 829ba5074bfaea0a584b78fdef406afb011bd697 Mon Sep 17 00:00:00 2001 From: Andrei Stepanov Date: Aug 16 2018 14:28:28 +0000 Subject: Merge #240 `Support for a simple fmf filter in beakerlib role` --- diff --git a/roles/standard-test-beakerlib/README.md b/roles/standard-test-beakerlib/README.md index 9b69440..09897f5 100644 --- a/roles/standard-test-beakerlib/README.md +++ b/roles/standard-test-beakerlib/README.md @@ -6,6 +6,7 @@ testing. You can redefine the following variables in `roles/standard-test-beakerlib/vars/main.yml`: * **tests**: A list of beakerlib test files + * **fmf_filter**: An fmf filter to select tests based on test metadata [1] * **artifacts**: An artifacts directory on localhost to store logs * **remote_artifacts**: The directory on the system under test where the logs are stored. Note: if this variable is left undefined, it will default to @@ -21,6 +22,7 @@ testing. You can redefine the following variables in url ... repository url (required) dest ... destination directory (required) version ... desired branch (optional, defaults to master) + fmf_filter ... fmf filter (optional, used for test selection) Example usage: @@ -39,3 +41,30 @@ Example usage: required_packages: - expect - which + +Instead of manually listing all tests to be executed it is also +possible to provide an fmf [1] filter in the following way: + + - hosts: localhost + roles: + - role: standard-test-beakerlib + tags: + - classic + repositories: + - repo: "https://src.fedoraproject.org/tests/shell.git" + dest: "shell" + fmf_filter: "tier: 1" + +Filter can be used also if tests are stored directly in the git: + + - hosts: localhost + roles: + - role: standard-test-beakerlib + tags: + - classic + fmf_filter: "tier: 1" + +See the Fedora CI metadata wiki [1] for more details and examples +on test metadata. + +[1] https://fedoraproject.org/wiki/CI/Metadata diff --git a/roles/standard-test-beakerlib/tasks/main.yml b/roles/standard-test-beakerlib/tasks/main.yml index 43b9d4a..ef61fd2 100644 --- a/roles/standard-test-beakerlib/tasks/main.yml +++ b/roles/standard-test-beakerlib/tasks/main.yml @@ -5,6 +5,7 @@ with_flattened: # tests - is defined at tests.yml playbook - "{{ tests }}" + - "{{ filter_tests }}" register: deps delegate_to: "{{ test_runner_inventory_name }}" delegate_facts: True @@ -112,6 +113,7 @@ fi with_items: - "{{ tests }}" + - "{{ filter_tests }}" always: - name: Make the master tests summary log artifact @@ -134,6 +136,7 @@ mv ${logfile} {{ remote_artifacts }}/${status}_${log_file_name} with_items: - "{{ tests }}" + - "{{ filter_tests }}" # Can't go in block. See # https://github.com/ansible/ansible/issues/20736 diff --git a/roles/str-common-init/tasks/trunner.yml b/roles/str-common-init/tasks/trunner.yml index b14fca5..d93d656 100644 --- a/roles/str-common-init/tasks/trunner.yml +++ b/roles/str-common-init/tasks/trunner.yml @@ -43,6 +43,43 @@ "{{ repositories }}" when: repositories is defined + # Prepare the filter command (check 'filter' variable for tests + # stored directly in the repo and also 'filter' key for each + # fetched test repo dictionary in the 'repositories' list). + - set_fact: + filter_command: "str-filter-tests" + - set_fact: + filter_command: "{{ filter_command }} . '{{ fmf_filter }}'" + when: fmf_filter is defined + - set_fact: + filter_command: "{{ filter_command }} '{{ item.dest }}' '{{ item.fmf_filter }}'" + with_items: + - "{{ repositories }}" + when: repositories is defined and item.fmf_filter is defined + - debug: + var: filter_command + verbosity: 1 + + # Filter tests based on the gathered parameters. (The filter + # command is extended for each provided filter. That's why + # there's check for filter_command != "str-filter-tests" which + # ensures that there has been at least one filter defined.) + - set_fact: + filter_tests: [] + - name: Filter tests according to provided fmf filters + shell: "{{ filter_command }}" + register: filter_output + when: filter_command != "str-filter-tests" + - debug: + var: filter_output + verbosity: 2 + - set_fact: + filter_tests: "{{ filter_output.stdout_lines }}" + when: filter_output.stdout_lines is defined + - debug: + var: filter_tests + verbosity: 1 + - debug: msg: > {{ test_runner_inventory_name }} is: diff --git a/scripts/README.md b/scripts/README.md index bf5da45..fb8c642 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,3 +28,12 @@ environment variables set in the inventory script. export TEST_DOCKER_EXTRA_ARGS="--cap-add=SYS_ADMIN" exec merge-standard-inventory "$@" ``` + +## `str-filter-tests` + +This script is used to filter tests for execution based on their +metadata stored in the Flexible Metadata Format [1]. It accepts +pairs of the form "path filter" for each metadata tree to be +investigated. Returns list of filtered tests on the stdout. + +[1] https://github.com/psss/fmf diff --git a/scripts/str-filter-tests b/scripts/str-filter-tests new file mode 100755 index 0000000..f9a4085 --- /dev/null +++ b/scripts/str-filter-tests @@ -0,0 +1,40 @@ +#!/bin/env python +""" +Filter tests based on metadata and provided path and filter. +Usage: {0} path filter [path filter...] +Directories with no metadata present are silently ignored. +""" + +from __future__ import unicode_literals + +import fmf +import sys + + +def usage(): + """ Show usage message and exit """ + print(__doc__.format(sys.argv[0])) + raise SystemExit(1) + + +if __name__ == "__main__": + # Check for correct arguments + pairs = sys.argv[1:] + if len(pairs) < 2: + usage() + + # Filter tests for each path-filter pair + while pairs: + try: + path = pairs.pop(0).rstrip('/') + filtr = pairs.pop(0) + except IndexError: + usage() + + try: + tree = fmf.Tree(path) + except fmf.utils.RootError: + fmf.utils.info("No metadata found for '{0}'.".format(path)) + continue + for node in tree.prune(keys=['test'], filters=[filtr]): + print(path + (node.data.get('path') or node.name))