#35 Add a new runtests and combine it with upload script
Opened a year ago by jcline. Modified a year ago
jcline/kernel-tests update-test-submission  into  master

file removed
-55
@@ -1,55 +0,0 @@ 

- #!/usr/bin/env python3

- #

- # Licensed under the terms of the GNU GPL License version 2

- 

- from __future__ import print_function

- from fedora.client import OpenIdBaseClient

- import getpass

- import sys, getopt

- 

- username = ''

- password = ''

- log = ''

- 

- argv = sys.argv[1:]

- try:

-     opts, args = getopt.getopt(argv,"hu:p:l:",["user=","password=", "logfile="])

- except getopt.GetoptError:

-     print('fedora_submit.py -u <fasuser> [-p <password>] -l <logfile>')

-     sys.exit(2)

- for opt, arg in opts:

-     if opt == '-h':

-         print('fedora_submit.py -u <fasuser> [-p <password>] -l <logfile>')

-         sys.exit()

-     elif opt in ("-u", "--user"):

-         username = arg

-     elif opt in ("-p", "--password"):

-         password = arg

-     elif opt in ("-l", "--logfile"):

-         log = arg

- 

- if username == '' or log == '':

-     print('fedora_submit.py -u <fasuser> [-p <password>] -l <logfile>')

-     sys.exit(2)

- if password == '':

-     password = getpass.getpass('FAS password: ')

- 

- submitclient = OpenIdBaseClient(

-     base_url='https://apps.fedoraproject.org/kerneltest/',

-     login_url='https://apps.fedoraproject.org/kerneltest/login',

-     username=username,

- )

-  

- submitclient.login(

-     submitclient.username,

-     password=password

- )

-  

- req = submitclient.send_request(

-     'https://apps.fedoraproject.org/kerneltest/upload/anonymous',

-     verb='POST',

-     auth=True,

-     files= { 'test_result': ('logfile', open(log, 'rb'), 'text/x-log'),}

- )

-  

- print(req.message)

file added
+183
@@ -0,0 +1,183 @@ 

+ #!/usr/bin/env python3

+ 

+ import argparse

+ import subprocess

+ import json

+ import os

+ import sys

+ import time

+ 

+ from openidc_client import OpenIDCClient, requestsauth

+ import requests

+ 

+ BASE_URL = "https://apps.fedoraproject.org/kerneltest"

+ ID_PROVIDER = "https://id.fedoraproject.org/openidc/"

+ APP_IDENTIFIER = "kerneltest"

+ SCOPES = [

+     "openid",

+     "https://github.com/jmflinuxtx/kerneltest-harness/oidc/upload_test_run",

+ ]

+ ID_PROVIDER_MAPPING = {

+     "Authorization": "Authorization",

+     "Token": "Token",

+     "UserInfo": "UserInfo",

+ }

+ CLIENT_ID = "kerneltest"

+ CLIENT_SECRET = "notsecret"

+ CACHEDIR = "~/.cache/kerneltest"

+ 

+ 

+ def upload_results(results, anonymous=False):

+     client = OpenIDCClient(

+         APP_IDENTIFIER,

+         ID_PROVIDER,

+         ID_PROVIDER_MAPPING,

+         CLIENT_ID,

+         client_secret=CLIENT_SECRET,

+         cachedir=CACHEDIR,

+     )

+     auth = None if anonymous else requestsauth.OpenIDCClientAuther(client, SCOPES)

+     response = requests.post(BASE_URL + "/api/v1/results/", auth=auth, json=results)

+     if response.status_code != 201:

+         print(response.status_code)

+         print(response.text)

+ 

+ 

+ parser = argparse.ArgumentParser(description="Run and submit kernel tests")

+ parser.add_argument("--logfile", help="The log file from a previous test run")

+ parser.add_argument(

+     "--upload", action="store_true", help="Upload the results to Fedora's servers"

+ )

+ parser.add_argument(

+     "--anonymous", action="store_true", help="Upload the results anonymously"

+ )

+ parser.add_argument(

+     "--minimal", action="store_true", help="Run only the minimal test suite"

+ )

+ parser.add_argument("--stress", action="store_true", help="Run the stress test suite")

+ parser.add_argument(

+     "--performance", action="store_true", help="Run the performance test suite"

+ )

+ parser.add_argument(

+     "--destructive", action="store_true", help="Run the destructive test suite"

+ )

+ parser.add_argument(

+     "--thirdparty", action="store_true", help="Run the third-party test suite"

+ )

+ parser.add_argument(

+     "--secureboot", action="store_true", help="Check the Secure Boot signature"

+ )

+ parser.add_argument("--disable-retest", help="Disable re-testing.")

+ args = parser.parse_args()

+ 

+ 

+ if args.logfile:

+     with open(args.logfile) as fd:

+         results = json.load(fd)

+     try:

+         upload_results(results, args.anonymous)

+         sys.exit(0)

+     except Exception:

+         sys.exit(1)

+ 

+ encoding = sys.getdefaultencoding()

+ 

+ result = subprocess.run(["uname", "-r"], encoding=encoding, capture_output=True)

+ rpm_version = result.stdout.strip()

+ kernel_version, release_and_arch = rpm_version.split("-")

+ build_release, arch = release_and_arch.rsplit(".", 1)

+ with open("/etc/os-release") as fd:

+     for line in fd.readlines():

+         if line.startswith("VERSION_ID="):

+             fedora_version = int(line.split("=")[1])

+             break

+     else:

+         raise Exception("No VERSION_ID found in /etc/os-release")

+ 

+ test_results = {

+     "kernel_version": kernel_version,

+     "build_release": build_release,

+     "arch": arch,

+     "fedora_version": fedora_version,

+     "tests": [],

+ }

+ 

+ 

+ directories = ["minimal", "default"]

+ if args.minimal:

+     directories = ["minimal"]

+ if args.destructive:

+     directories.append("destructive")

+ if args.performance:

+     directories.append("performance")

+ if args.secureboot:

+     directories.append("secureboot")

+ if args.stress:

+     directories.append("stress")

+ if args.thirdparty:

+     directories.append("thirdparty")

+ if args.stress:

+     directories.append("stress")

+ 

+ # run test, check return code

+ # 0 = pass

+ # 3 = skip

+ # 4 - warn

+ # anything else is fail

+ root_dir = os.getcwd()

+ for directory in directories:

+     for dirpath, dirnames, filenames in os.walk(directory):

+         if "runtest.sh" in filenames:

+             os.chdir(dirpath)

+             print("Running " + dirpath)

+             result = subprocess.run(

+                 ["./runtest.sh"], encoding=encoding, shell=True, capture_output=True

+             )

+             os.chdir(root_dir)

+             test_results["tests"].append(

+                 {

+                     "name": dirpath,

+                     "passed": result.returncode == 0,

+                     "waived": result.returncode == 3,

+                     "details": result.stdout,

+                 }

+             )

+ 

+ logdir = os.path.join(root_dir, "logs", str(fedora_version), rpm_version)

+ logfile = os.path.join(

+     logdir, "{suite}-{date}.log".format(suite="-".join(directories), date=time.time())

+ )

+ 

+ os.makedirs(logdir, exist_ok=True)

+ with open(logfile, "w") as fd:

+     json.dump(test_results, fd, sort_keys=True, indent=4)

+ 

+ if args.upload:

+     try:

+         upload_results(test_results, args.anonymous)

+     except Exception as e:

+         print("Failed to upload results: " + str(e))

+ 

+ print("Your log file is located at: " + logfile)

+ print("Submit your results to: https://apps.fedoraproject.org/kerneltest/")

+ 

+ print("The following information is not submitted with your log;")

+ print("it is for informational purposes only.")

+ 

+ if os.path.exists("/usr/bin/pesign"):

+     sign_result = subprocess.run(

+         ["pesign", "-i", "/boot/vmlinuz-{}".format(rpm_version), "-S"]

+     )

+     print(sign_result)

+ 

+ print("Vulnerability status:")

+ vulns = "/sys/devices/system/cpu/vulnerabilities/"

+ for vuln in os.listdir(vulns):

+     with open(os.path.join(vulns, vuln)) as fd:

+         print("{}: {}".format(vuln, fd.read().strip()))

+ 

+ 

+ if all([test["passed"] for test in test_results["tests"]]):

+     sys.exit(0)

+ else:

+     sys.exit(1)

file removed
-211
@@ -1,211 +0,0 @@ 

- #!/bin/bash

- #

- # Licensed under the terms of the GNU GPL License version 2.

- 

- date=$(date +%s)

- logdir=$(pwd)/logs

- logfile=$logdir/kernel-test-$date.log.txt

- verbose=n

- testset=default

- cleanrun=PASS

- failedtests=None

- warntests=None

- commit=n

- commithook=/usr/bin/true

- disable_retest=n

- thirdparty=n

- 

- if [ -f ./.config ]; then

- 	source ./.config

- else

- 	echo "No .config file found. You can cp config.example .config and edit as needed for easier log submission."

- fi

- 

- kver=$(uname -r)

- release=$(cat /etc/redhat-release)

- 

- # Check for pre-requisites.

- if [ ! -f /usr/bin/gcc ]; then

- 	echo " Could not locate gcc. The Fedora kernel test suite requires gcc."

- 	exit

- fi

- 

- # Unset MALLOC_CHECK_ and MALLOC_PERTURB_. Some tests might not work

- # well with those active (e.g. libhugetlbfs).

- unset MALLOC_CHECK_

- unset MALLOC_PERTURB_

- 

- if [ ! -d "$logdir" ] ; then

- 	mkdir $logdir

- fi

- 

- args=y

- 

- while [ $args = y ]

- do

-         case "$1" in

- 	-v)

- 		#TO DO: Implement verbose behavior.

- 		verbose=y

- 		shift 1

- 		;;

- 	-t)

- 		testset=$2

- 		shift 2

- 		;;

- 	*)

- 		args=n

- 	esac

- done

- 

- 

- if [ "$disable_retest" == "y" ]; then

-         # Check if wanted test has been executed with current kernel

-         # version.

-         for file in $(find $logdir -name \*.log.txt); do

-                 version_tested=$(cat $file | sed -n 3p | cut -d ' ' -f 2)

-                 test_executed=$(cat $file | sed -n 2p | cut -d ' ' -f 3)

- 

-                 if [ "$version_tested" == "$kver" -a "$test_executed" == "$testset" ]; then

-                         echo "The current kernel was already tested with this test, abort."

-                         exit 0

-                 fi

-         done

- fi

- 

- case $testset in

- minimal)

- 	dirlist="minimal"

- 	;;

- default)

- 	dirlist="minimal default"

- 	;;

- stress)

- 	dirlist="minimal default stress"

- 	;;

- destructive)

- 	echo "You have specified the destructive test set."

- 	echo "This test may cause damage to your system."

- 	echo "Are you sure that you wish to continue?"

- 	read continue

- 	if [ $continue == 'y' ] ; then

- 		dirlist="minimal default destructive"

- 	else

- 		dirlist="minimal default"

- 		testset=default

- 	fi

- 	;;

- performance)

- 	dirlist="performance"

- 	;;

- *)

- 	echo "Supported test sets are minimal, default, stress, destructive or performance."

- 	exit 1

- esac

- 

- # Test Secure Boot?

- if  [ "$checksig" == "y" ]; then

-     dirlist="secureboot $dirlist"

- fi

- 

- # Test Third Party Modules?

- if  [ "$thirdparty" == "y" ]; then

-     dirlist="$dirlist thirdparty"

- fi

- 

- # Basic logfile headers.

- echo "Date: $(date)" > $logfile

- echo "Test set: $testset" >> $logfile

- echo "Kernel: $kver" >> $logfile

- echo "Release: $release" >> $logfile

- echo "Result: RESULTHOLDER" >> $logfile

- echo "============================================================" >>$logfile

- 

- 

- 

- # Start running tests.

- echo "Test suite called with $testset"

- 

- for dir in $dirlist

- do

- 	for test in $(find ./$dir/ -name runtest.sh)

- 	do

- 		testdir=$(dirname $test)

- 		pushd $testdir &>/dev/null

- 		#TO DO:  Purpose file test name format.

- 		testname=$testdir

- 		echo "Starting test $testname" >> $logfile

- 

- 		if [ "$testset" == "performance" ]; then

- 			./runtest.sh >>$logfile

- 		elif [ "$dir" == "secureboot" ]; then

- 			./runtest.sh "$validsig" &>>$logfile

- 		else

- 			./runtest.sh &>>$logfile

- 		fi

- 		complete=$?

- 		case $complete in

- 		0)

- 			result=PASS

- 			;;

- 		3)

- 			result=SKIP

- 			;;

- 		4)

- 			result=WARN

- 			;;

- 		*)

- 			result=FAIL

- 		esac

- 		printf "%-65s%-8s\n" "$testname" "$result"

- 		if [ "$result" == "FAIL" ]; then

- 			cleanrun=FAIL

- 			if [ "$failedtests" == "None" ]; then

- 				failedtests="$testname"

- 			else

- 				failedtests="$failedtests $testname"

- 			fi

- 		fi

- 		if [ "$result" == "WARN" ]; then

- 			if [ "$cleanrun" == "PASS" ]; then

- 				cleanrun=WARN

- 			fi

- 			if [ "$warntests" == "None" ]; then

- 				warntests="$testname"

- 			else

- 				warntests="$warntests $testname"

- 			fi

- 		fi

- 		popd &>/dev/null

- 	done

- done

- 

- # Fix up logfile headers.

- sed -i "s,RESULTHOLDER,$cleanrun\nFailed Tests: $failedtests\nWarned Tests: $warntests,g" $logfile

- printf "\n%-65s%-8s\n" "Test suite complete" "$cleanrun"

- 

- if [ "$commit" == "y" ]; then

- 	printf "\nYour log file is being submitted...\n"

- 	$commithook

- else

- 	printf "\nYour log file is located at: $logfile\n"

- 	printf "Submit your results to: https://apps.fedoraproject.org/kerneltest/\n"

- fi

- 

- echo "The following information is not submitted with your log;"

- echo "it is for informational purposes only."

- 

- if [ -f /usr/bin/pesign ]; then

- 	echo "Checking for kernel signature:"

- 	/usr/bin/pesign -i /boot/vmlinuz-$kver -S | grep "common name"

- fi

- 

- echo "Vulnerability status:"

- grep . /sys/devices/system/cpu/vulnerabilities/*

- 

- if [ "$cleanrun" == "FAIL" ]; then

- 	exit 1

- else

- 	exit 0

- fi

- 

I have no memory of writing this so I don't recall if it works or not, but in theory this should submit results to the version of kerneltests-harness that supports OpenID Connect.

Not sure what your intent is, but I merged your pull request and ran sudo ./runtests. My test results were not uploaded. When I run runtests.sh before this change, my results are automatically uploaded. (I also see an indication of pass/fail on the console when I run runtests.sh before this change.

Maybe I do not understand the purpose of the pull request.

garry@tfr$ sudo ./runtests
[sudo] password for garry:
Running default/memfd
Running default/insert_leap_second
Running default/selinux-dac-controls
Running default/modsign
Running default/mq-memory-corruption
Running default/paxtest
Running default/libhugetlbfs
Running default/sysfs-perms
Running default/cachedrop
Running default/posix_timers
Running default/stack-randomness
Running default/timer-overhead
Your log file is located at: /home/garry/src/kernel-tests/logs/32/5.6.10-300.fc32.x86_64/minimal-default-1588903174.0848687.log
Submit your results to: https://apps.fedoraproject.org/kerneltest/
The following information is not submitted with your log;
it is for informational purposes only.
Vulnerability status:
spectre_v2: Mitigation: Full generic retpoline, IBPB: conditional, IBRS_FW, STIBP: conditional, RSB filling
itlb_multihit: KVM: Mitigation: Split huge pages
mds: Mitigation: Clear CPU buffers; SMT vulnerable
l1tf: Mitigation: PTE Inversion; VMX: conditional cache flushes, SMT vulnerable
spec_store_bypass: Mitigation: Speculative Store Bypass disabled via prctl and seccomp
tsx_async_abort: Not affected
spectre_v1: Mitigation: usercopy/swapgs barriers and __user pointer sanitization
meltdown: Mitigation: PTI
garry@tfr$

That is because this pull request has nothing to do with the code that is currently running on https://apps.fedoraproject.org/kerneltest/
It is required for a rewrite of the app which is currently in staging. Provided testing works out, this code would be merged at the time that the front end code is promoted from staging.