#347 support to timeout
Merged 4 years ago by astepano. Opened 4 years ago by bgoncalv.
bgoncalv/standard-test-roles fix-role  into  master

@@ -18,6 +18,12 @@ 

   * **required_services**: A list of services to be started before running tests.

     Please use `required_packages` for specifying packages that provide the services.

  

+ ## Test case parameters

+ 

+  * **dir**: test directory. default: is the test name

+  * **run**: command to run the test. default: ./runtest.sh

+  * **timeout**: abort test case after this time. More details on [timeout][1]. default: 0

+ 

  Example usage:

  

      - hosts: localhost
@@ -35,6 +41,7 @@ 

          - smoke37:

              dir: python/smoke

              run: VERSION=3.7 ./venv.sh

+             timeout: 1h

          required_packages:

          - python27

          - python37
@@ -42,3 +49,5 @@ 

          - python3-virtualenv

          - python2-devel

          - python3-devel

+ 

+ [1]: http://man7.org/linux/man-pages/man1/timeout.1.html

@@ -23,6 +23,7 @@ 

  -w, --workdir           test environment work dir. Directory for test execution.

  -a, --artifactsdir      test environment dir to store artifacts

  -t, --testname          name of the test

+     --timeout           test timeout

  -c, --cmd               shell command to run the test

  EOF

  }
@@ -37,9 +38,10 @@ 

  STR_WORKDIR="${STR_WORKDIR:-}"

  STR_TEST_NAME="${STR_TEST_NAME:-}"

  STR_ARTIFACTS_DIR="${STR_ARTIFACTS_DIR:-/tmp}"

+ STR_TIMEOUT="${STR_TIMEOUT:-0}"

  

  # http://wiki.bash-hackers.org/howto/getopts_tutorial

- opt=$(getopt -n "$0" --options "hvt:w:a:c:" --longoptions "help,verbose,cmd:,testname:,workdir:,artifactsdir:" -- "$@")

+ opt=$(getopt -n "$0" --options "hvt:w:a:c:" --longoptions "help,verbose,cmd:,testname:,workdir:,artifactsdir:,timeout:" -- "$@")

  eval set -- "$opt"

  while [[ $# -gt 0 ]]; do

      case "$1" in
@@ -59,6 +61,10 @@ 

              STR_ARTIFACTS_DIR="$2"

              shift 2

              ;;

+         --timeout)

+             STR_TIMEOUT="$2"

+             shift 2

+             ;;

          -v|--verbose)

              DEBUG="-v"

              shift
@@ -91,6 +97,7 @@ 

  debug "Command: $STR_CMD"

  debug "Work dir: $STR_WORKDIR"

  debug "Artifacts dir: $STR_ARTIFACTS_DIR"

+ debug "Timeout: $STR_TIMEOUT"

  

  # Up to this point any fail is considered as ci-sytem fail. Exit code != 0.

  # Starting from this point and bellow any fail is considered as a test fail. Exit code == 0.
@@ -111,6 +118,10 @@ 

      if [[ $rc -eq 127 ]]; then

          echo "$STR_TEST_NAME (problem with test execution)" >&2

          status="ERROR"

+     elif [[ $rc -eq 124 ]]; then

+         # test case timed out

+         echo "$STR_TEST_NAME (test aborted due to timeout)" >&2

+         status="ERROR"

      elif [[ $rc -eq 0 ]]; then

          status="PASS"

      fi
@@ -143,6 +154,6 @@ 

  if [ -f "$STR_CMD" ]; then

      chmod 0775 "$STR_CMD"

  fi

- bash -c "$STR_CMD"

+ timeout "$STR_TIMEOUT" bash -c "$STR_CMD"

  # Explicit return code

  exit $?

@@ -18,6 +18,7 @@ 

                  --workdir "{{ tenv_workdir }}/{{ item if item.keys is not defined else item[(item.keys()|list)[0]]['dir']|default((item.keys()|list)[0]) }}" \

                  --artifactsdir "{{ remote_artifacts }}" \

                  --test "{{ item if item.keys is not defined else (item.keys()|list)[0] }}" \

+                 --timeout "{{ '0' if item.keys is not defined else item[(item.keys()|list)[0]]['timeout']|default('0') }}" \

                  --cmd "{{ './runtest.sh' if item.keys is not defined else item[(item.keys()|list)[0]]['run']|default('./runtest.sh') | regex_replace('\\', '\\\\') | regex_replace('\"', '\"') | regex_replace('\$', '\\$') }}"

      with_items:

      - "{{ tests }}"

@@ -24,6 +24,10 @@ 

       version ... desired branch (optional, defaults to master)

       fmf_filter ... fmf filter (optional, used for test selection)

  

+ ## Test case parameters

+ 

+  * **timeout**: abort test case after this time. More details on [timeout][1]. default: 0

+ 

  Example usage:

  

  ```yaml
@@ -38,7 +42,8 @@ 

      tests:

      - shell/func

      - shell/login

-     - shell/smoke

+     - shell/smoke:

+         timeout: 1h

      required_packages:

      - expect

      - which

@@ -23,6 +23,7 @@ 

  -w, --workdir           test environment work dir

  -a, --artifactsdir      test environment dir to store artifacts

  -t, --test              test to run, can be a path to directory or a runtest.sh file

+     --timeout           test timeout

  EOF

  }

  
@@ -35,9 +36,10 @@ 

  STR_BKR_TEST="${STR_BKR_TEST:-}"

  STR_WORKDIR="${STR_WORKDIR:-}"

  STR_ARTIFACTS_DIR="${STR_ARTIFACTS_DIR:-}"

+ STR_TIMEOUT="${STR_TIMEOUT:-0}"

  

  # http://wiki.bash-hackers.org/howto/getopts_tutorial

- opt=$(getopt -n "$0" --options "hvt:w:a:" --longoptions "help,verbose,test:,workdir:,artifactsdir:" -- "$@")

+ opt=$(getopt -n "$0" --options "hvt:w:a:" --longoptions "help,verbose,test:,workdir:,artifactsdir:,timeout:" -- "$@")

  eval set -- "$opt"

  while [[ $# -gt 0 ]]; do

      case "$1" in
@@ -53,6 +55,10 @@ 

              STR_ARTIFACTS_DIR="$2"

              shift 2

              ;;

+         --timeout)

+             STR_TIMEOUT="$2"

+             shift 2

+             ;;

          -v|--verbose)

              DEBUG="-v"

              shift
@@ -80,6 +86,7 @@ 

  debug "Test: $STR_BKR_TEST"

  debug "Work dir: $STR_WORKDIR"

  debug "Artifacts dir: $STR_ARTIFACTS_DIR"

+ debug "Timeout: $STR_TIMEOUT"

  STR_BKR_TEST_DASHED=$(echo "$STR_BKR_TEST" | sed -e 's/\//-/g')

  

  # Up to this point any fail is considered as ci-sytem fail. Exit code != 0.
@@ -101,6 +108,10 @@ 

      local status

      if [[ $rc -eq 127 ]]; then

          status="ERROR"

+     elif [[ $rc -eq 124 ]]; then

+         # test case timed out

+         echo "$STR_BKR_TEST (test aborted due to timeout)" >&2

+         status="ERROR"

      elif grep -q "RESULT: WARN" "$log_file_path"; then

          status="ERROR"

      elif grep -q "RESULT: FAIL" "$log_file_path"; then
@@ -146,7 +157,7 @@ 

  if [ -f "$STR_BKR_TEST" ]; then

      debug "Running test from file: $STR_BKR_TEST"

      cd $(dirname "$STR_BKR_TEST")

-     /bin/sh -e ./$(basename "$STR_BKR_TEST") || :

+     timeout "$STR_TIMEOUT" /bin/sh -e ./$(basename "$STR_BKR_TEST")

      exit

  fi

  
@@ -157,10 +168,10 @@ 

      get-test-deps -i .

      if [ -f "Makefile" ] && command -p -v "make" >"/dev/null" 2>&1; then

          debug "Running test from Makefile"

-         make run || :

+         timeout "$STR_TIMEOUT" make run

      elif [ -f "runtest.sh" ]; then

          debug "Running test from runtest.sh"

-         /bin/sh -e ./runtest.sh || :

+         timeout "$STR_TIMEOUT" /bin/sh -e ./runtest.sh

      else

          echo "FAIL test $STR_BKR_TEST do not know how to run test"

      fi

@@ -52,7 +52,10 @@ 

  

  - block:

    - name: Run beakerlib tests

-     script: run-beakerlib-test --workdir {{ tenv_workdir }} --artifactsdir {{ remote_artifacts }} --test {{ item }}

+     script: run-beakerlib-test --workdir {{ tenv_workdir }} \

+                     --artifactsdir {{ remote_artifacts }} \

+                     --test "{{ item if item.keys is not defined else (item.keys()|list)[0] }}" \

+                     --timeout "{{ '0' if item.keys is not defined else item[(item.keys()|list)[0]]['timeout']|default('0') }}"

      with_items:

      - "{{ tests }}"

      - "{{ filter_tests }}"

file modified
+35
@@ -57,3 +57,38 @@ 

    - import_tasks: shared-tasks/verify_failed_test.yml

    - import_tasks: shared-tasks/artifacts_test_env.yml

    - import_tasks: shared-tasks/artifacts_test_runner.yml

+ 

+ # Make sure the role passes on timeout

+ - hosts: localhost

+   tags:

+   - atomic

+   - classic

+   - container

+   roles:

+   - role: standard-test-basic

+     tests:

+     - test-basic-timeout:

+         run: ls; sleep 1

+         timeout: 5

+   tasks:

+   - import_tasks: shared-tasks/artifacts_test_env.yml

+   - import_tasks: shared-tasks/artifacts_test_runner.yml

+ 

+ # Make sure the role fails on timeout

+ - hosts: localhost

+   tags:

+   - atomic

+   - classic

+   - container

+   roles:

+   - role: standard-test-basic

+     tests:

+     - test-basic-timeout-fail:

+         run: ls; sleep 5

+         timeout: 1

+     ignore_errors: yes

+   tasks:

+   # 'verify_error_test' tasks should run after 'test-basic-timeout-fail'

+   - import_tasks: shared-tasks/verify_error_test.yml

+   - import_tasks: shared-tasks/artifacts_test_env.yml

+   - import_tasks: shared-tasks/artifacts_test_runner.yml

file modified
+33
@@ -33,3 +33,36 @@ 

    - import_tasks: shared-tasks/verify_failed_test.yml

    - import_tasks: shared-tasks/artifacts_test_env.yml

    - import_tasks: shared-tasks/artifacts_test_runner.yml

+ 

+ # Make sure the role behaves correctly with timeout

+ - hosts: localhost

+   tags:

+   - atomic

+   - classic

+   - container

+   roles:

+   - role: standard-test-beakerlib

+     tests:

+     - test-beakerlib-timeout:

+         timeout: 10

+   tasks:

+   - import_tasks: shared-tasks/artifacts_test_env.yml

+   - import_tasks: shared-tasks/artifacts_test_runner.yml

+ 

+ # Make sure the role behaves correctly if timeout fails

+ - hosts: localhost

+   tags:

+   - atomic

+   - classic

+   - container

+   roles:

+   - role: standard-test-beakerlib

+     tests:

+     - test-beakerlib-timeout:

+         timeout: 1

+     ignore_errors: yes

+   tasks:

+   # 'tests_verify_error_test' tasks should run after 'test-beakerlib-timeout'

+   - import_tasks: shared-tasks/verify_error_test.yml

+   - import_tasks: shared-tasks/artifacts_test_env.yml

+   - import_tasks: shared-tasks/artifacts_test_runner.yml

@@ -0,0 +1,11 @@ 

+ # These tasks should run after as test that fails

+ - name: Read test.log on test environment

+   shell: "cat {{remote_artifacts}}/test.log"

+   register: test_log

+ 

+ - name: Check for ERROR on test.log on test environment

+   fail: msg="Could not find ERROR on test log"

+   when: test_log.stdout.find("ERROR") == -1

+   # We should fail when we can not find ERROR string on log

+   # -1 means string not found

+ 

@@ -0,0 +1,12 @@ 

+ #!/bin/bash

+ # Include Beaker environment

+ . /usr/share/beakerlib/beakerlib.sh || exit 1

+ 

+ PACKAGE="bash"

+ 

+ rlJournalStart

+     rlPhaseStartTest "beakerlib timeout test"

+         rlRun "ls /; sleep 5"

+     rlPhaseEnd

+ rlJournalPrintText

+ rlJournalEnd