From 9d87c81655ee5f37b0a49585a7774cd9ba6b8a58 Mon Sep 17 00:00:00 2001
From: Andrei Stepanov <astepano@redhat.com>
Date: May 07 2019 11:10:32 +0000
Subject: Merge #306 `Add support for results.yml [fix #296]`


---

diff --git a/roles/standard-test-basic/files/run-basic-test b/roles/standard-test-basic/files/run-basic-test
index 60190ea..5be3556 100755
--- a/roles/standard-test-basic/files/run-basic-test
+++ b/roles/standard-test-basic/files/run-basic-test
@@ -106,16 +106,21 @@ clean_exit() {
             kill -s HUP $pid
         fi
     done
+    local status="FAIL"
     # Return non-zero when test command not found
     if [[ $rc -eq 127 ]]; then
         echo "$STR_TEST_NAME (problem with test execution)" >&2
-    fi
-    local status="FAIL"
-    if [[ $rc -eq 0 ]]; then
+        status="ERROR"
+    elif [[ $rc -eq 0 ]]; then
         status="PASS"
     fi
     local run_journal="$STR_ARTIFACTS_DIR/test.log"
     echo "${status} $STR_TEST_NAME" >> "$run_journal"
+    # Handle results.yml file
+    local results="$STR_ARTIFACTS_DIR/results.yml"
+    local result=$(echo $status | tr '[:upper:]' '[:lower:]')
+    test -f "$results" || echo 'results:' > "$results"
+    echo "- {result: $result, test: $STR_TEST_NAME}" >> "$results"
     if [ -e "$logfile_stdout" ]; then
         local new_logfile_stdout="$(dirname "$logfile_stdout")/${status}_$(basename "$logfile_stdout")"
         mv -f "$logfile_stdout" "$new_logfile_stdout"
diff --git a/roles/standard-test-basic/tasks/main.yml b/roles/standard-test-basic/tasks/main.yml
index 1985f33..83b4e55 100644
--- a/roles/standard-test-basic/tasks/main.yml
+++ b/roles/standard-test-basic/tasks/main.yml
@@ -25,15 +25,15 @@
   # Can't go in block. See
   # https://github.com/ansible/ansible/issues/20736
   - name: Check the results
-    shell: grep "^FAIL" {{ remote_artifacts }}/test.log
-    register: test_fails
+    shell: grep "^ERROR" {{ remote_artifacts }}/test.log
+    register: test_error
     # Never fail at this step. Just store result of tests.
     failed_when: False
 
   - name: Set role result
     set_fact:
-      role_result_failed: "{{ (test_fails.stdout|d|length > 0) or (test_fails.stderr|d|length > 0) }}"
-      role_result_msg: "{{ test_fails.stdout|d('tests failed.') }}"
+      role_result_error: "{{ (test_error.stdout|d|length > 0) or (test_error.stderr|d|length > 0) }}"
+      role_result_msg: "{{ test_error.stdout|d('test execution error.') }}"
 
   - include_role:
       name: str-common-final
diff --git a/roles/standard-test-beakerlib/files/run-beakerlib-test b/roles/standard-test-beakerlib/files/run-beakerlib-test
index 86bd722..a46bf4b 100755
--- a/roles/standard-test-beakerlib/files/run-beakerlib-test
+++ b/roles/standard-test-beakerlib/files/run-beakerlib-test
@@ -80,6 +80,7 @@ fi
 debug "Test: $STR_BKR_TEST"
 debug "Work dir: $STR_WORKDIR"
 debug "Artifacts dir: $STR_ARTIFACTS_DIR"
+STR_BKR_TEST_DASHED=$(echo "$STR_BKR_TEST" | sed -e 's/\//-/g')
 
 # Up to this point any fail is considered as ci-sytem fail. Exit code != 0.
 # Starting from this point and bellow any fail is considered as a test fail. Exit code == 0.
@@ -88,13 +89,39 @@ clean_exit() {
     rc=$?;
     trap - SIGINT SIGTERM SIGABRT EXIT # clear the trap
     echo "Run test $STR_BKR_TEST: done."
-    # Exit code == 0, no matter of the test result.
     # Close tee pipes
     for pid in $(ps -o pid --no-headers --ppid $$); do
         if [ -n "$(ps -p $pid -o pid=)" ]; then
             kill -s HUP $pid
         fi
     done
+    # Check test result status
+    local log_file_name="$STR_BKR_TEST_DASHED.log"
+    local log_file_path="$STR_ARTIFACTS_DIR/$log_file_name"
+    local status
+    if [[ $rc -eq 127 ]]; then
+        status="ERROR"
+    elif grep -q "RESULT: WARN" "$log_file_path"; then
+        status="ERROR"
+    elif grep -q "RESULT: FAIL" "$log_file_path"; then
+        status="FAIL"
+    elif grep -q "RESULT: PASS" "$log_file_path"; then
+        status="PASS"
+    elif grep -q "FAIL" "$log_file_path"; then
+        status="FAIL"
+    elif grep -q "PASS" "$log_file_path"; then
+        status="PASS"
+    else
+        status="ERROR"
+    fi
+    echo "$status $STR_BKR_TEST" >> "$STR_ARTIFACTS_DIR/test.log"
+    mv "$log_file_path" "$STR_ARTIFACTS_DIR/${status}_${log_file_name}"
+    # Handle results.yml file
+    local results="$STR_ARTIFACTS_DIR/results.yml"
+    local result=$(echo $status | tr '[:upper:]' '[:lower:]')
+    test -f "$results" || echo 'results:' > "$results"
+    echo "- {result: $result, test: $STR_BKR_TEST}" >> "$results"
+    # Exit code == 0, no matter of the test result.
     exit 0
 }
 trap clean_exit SIGINT SIGTERM SIGABRT EXIT
@@ -103,9 +130,9 @@ trap clean_exit SIGINT SIGTERM SIGABRT EXIT
 export PATH="$PATH:$STR_WORKDIR"
 mkdir -p "$STR_ARTIFACTS_DIR"
 # OUTPUTFILE has influence on beakerlib-libraries output
-export OUTPUTFILE="$(realpath "$STR_ARTIFACTS_DIR")/$(echo "$STR_BKR_TEST" | sed -e 's/\//-/g')-out.log"
-logfile_stdout="$STR_ARTIFACTS_DIR/$(echo "$STR_BKR_TEST" | sed -e 's/\//-/g').log"
-logfile_stderr="$STR_ARTIFACTS_DIR/$(echo "$STR_BKR_TEST" | sed -e 's/\//-/g')-err.log"
+export OUTPUTFILE="$(realpath "$STR_ARTIFACTS_DIR")/$STR_BKR_TEST_DASHED-out.log"
+logfile_stdout="$STR_ARTIFACTS_DIR/$STR_BKR_TEST_DASHED.log"
+logfile_stderr="$STR_ARTIFACTS_DIR/$STR_BKR_TEST_DASHED-err.log"
 exec 3>&1 4>&2 1> >(tee -a "$logfile_stdout" >&3) 2> >(tee -a "$logfile_stderr" >&4)
 mkdir -p "$STR_WORKDIR"
 cd "$STR_WORKDIR"
@@ -113,7 +140,7 @@ cd "$STR_WORKDIR"
 if ! [ -d "$STR_BKR_TEST" ] && ! [ -f "$STR_BKR_TEST" ]; then
     # Next string goes to .log file
     echo "FAIL test $STR_BKR_TEST does not appear to be a file or directory"
-    exit 1
+    exit 127
 fi
 
 if [ -f "$STR_BKR_TEST" ]; then
diff --git a/roles/standard-test-beakerlib/tasks/main.yml b/roles/standard-test-beakerlib/tasks/main.yml
index 4302e53..64e221b 100644
--- a/roles/standard-test-beakerlib/tasks/main.yml
+++ b/roles/standard-test-beakerlib/tasks/main.yml
@@ -60,40 +60,18 @@
       BEAKERLIB_LIBRARY_PATH: "{{ beakerlib_libraries_path }}"
 
   always:
-  - name: Make the master tests summary log artifact
-    shell: |
-      log_file_name=$(echo {{ item }} | sed -e 's/\//-/g').log
-      logfile={{ remote_artifacts }}/${log_file_name}
-      status="FAIL"
-      if grep -q '\[ *FAIL *\]' "$logfile"; then
-        status="FAIL"
-      elif grep -q '\[ *PASS *\]' "$logfile"; then
-        status="PASS"
-      elif grep -q FAIL "$logfile"; then
-        status="FAIL"
-      elif grep -q PASS "$logfile"; then
-        status="PASS"
-      else
-        status="FAIL"
-      fi
-      echo "${status} {{ item }}" >> {{ remote_artifacts }}/test.log
-      mv ${logfile} {{ remote_artifacts }}/${status}_${log_file_name}
-    with_items:
-    - "{{ tests }}"
-    - "{{ filter_tests }}"
-
   # Can't go in block. See
   # https://github.com/ansible/ansible/issues/20736
   - name: Check the results
-    shell: grep "^FAIL" {{ remote_artifacts }}/test.log
-    register: test_fails
+    shell: grep "^ERROR" {{ remote_artifacts }}/test.log
+    register: test_error
     # Never fail at this step. Just store result of tests.
     failed_when: False
 
   - name: Set role result
     set_fact:
-      role_result_failed: "{{ (test_fails.stdout|d|length > 0) or (test_fails.stderr|d|length > 0) }}"
-      role_result_msg: "{{ test_fails.stdout|d('Beaker tests failed.') }}"
+      role_result_error: "{{ (test_error.stdout|d|length > 0) or (test_error.stderr|d|length > 0) }}"
+      role_result_msg: "{{ test_error.stdout|d('test execution error.') }}"
 
   - include_role:
       name: str-common-final
diff --git a/roles/str-common-final/tasks/main.yml b/roles/str-common-final/tasks/main.yml
index 53989d4..55f4299 100644
--- a/roles/str-common-final/tasks/main.yml
+++ b/roles/str-common-final/tasks/main.yml
@@ -11,8 +11,8 @@
 - name: Report role result
   vars:
     msg: |
-       Tests failed: {{ role_result_failed|d('Undefined') }}
+       Tests error: {{ role_result_error|d('Undefined') }}
        Tests msg: {{ role_result_msg|d('None') }}
   debug:
     msg: "{{ msg.split('\n') }}"
-  failed_when: "role_result_failed|bool"
+  failed_when: "role_result_error|bool"