From 6037b635d564625f4649d21823efc2fcd456001e Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Jul 02 2019 15:22:00 +0000 Subject: Add a new runtests and combine it with upload script Signed-off-by: Jeremy Cline --- diff --git a/fedora_submit.py b/fedora_submit.py deleted file mode 100755 index 8b12e85..0000000 --- a/fedora_submit.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed under the terms of the GNU GPL License version 2 - -from __future__ import print_function -from fedora.client import OpenIdBaseClient -import getpass -import sys, getopt - -username = '' -password = '' -log = '' - -argv = sys.argv[1:] -try: - opts, args = getopt.getopt(argv,"hu:p:l:",["user=","password=", "logfile="]) -except getopt.GetoptError: - print('fedora_submit.py -u [-p ] -l ') - sys.exit(2) -for opt, arg in opts: - if opt == '-h': - print('fedora_submit.py -u [-p ] -l ') - sys.exit() - elif opt in ("-u", "--user"): - username = arg - elif opt in ("-p", "--password"): - password = arg - elif opt in ("-l", "--logfile"): - log = arg - -if username == '' or log == '': - print('fedora_submit.py -u [-p ] -l ') - sys.exit(2) -if password == '': - password = getpass.getpass('FAS password: ') - -submitclient = OpenIdBaseClient( - base_url='https://apps.fedoraproject.org/kerneltest/', - login_url='https://apps.fedoraproject.org/kerneltest/login', - username=username, -) - -submitclient.login( - submitclient.username, - password=password -) - -req = submitclient.send_request( - 'https://apps.fedoraproject.org/kerneltest/upload/anonymous', - verb='POST', - auth=True, - files= { 'test_result': ('logfile', open(log, 'rb'), 'text/x-log'),} -) - -print(req.message) diff --git a/runtests b/runtests new file mode 100755 index 0000000..c34fe01 --- /dev/null +++ b/runtests @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +import argparse +import subprocess +import json +import os +import sys +import time + +from openidc_client import OpenIDCClient, requestsauth +import requests + +BASE_URL = "https://apps.fedoraproject.org/kerneltest" +ID_PROVIDER = "https://id.fedoraproject.org/openidc/" +APP_IDENTIFIER = "kerneltest" +SCOPES = [ + "openid", + "https://github.com/jmflinuxtx/kerneltest-harness/oidc/upload_test_run", +] +ID_PROVIDER_MAPPING = { + "Authorization": "Authorization", + "Token": "Token", + "UserInfo": "UserInfo", +} +CLIENT_ID = "kerneltest" +CLIENT_SECRET = "notsecret" +CACHEDIR = "~/.cache/kerneltest" + + +def upload_results(results, anonymous=False): + client = OpenIDCClient( + APP_IDENTIFIER, + ID_PROVIDER, + ID_PROVIDER_MAPPING, + CLIENT_ID, + client_secret=CLIENT_SECRET, + cachedir=CACHEDIR, + ) + auth = None if anonymous else requestsauth.OpenIDCClientAuther(client, SCOPES) + response = requests.post(BASE_URL + "/api/v1/results/", auth=auth, json=results) + if response.status_code != 201: + print(response.status_code) + print(response.text) + + +parser = argparse.ArgumentParser(description="Run and submit kernel tests") +parser.add_argument("--logfile", help="The log file from a previous test run") +parser.add_argument( + "--upload", action="store_true", help="Upload the results to Fedora's servers" +) +parser.add_argument( + "--anonymous", action="store_true", help="Upload the results anonymously" +) +parser.add_argument( + "--minimal", action="store_true", help="Run only the minimal test suite" +) +parser.add_argument("--stress", action="store_true", help="Run the stress test suite") +parser.add_argument( + "--performance", action="store_true", help="Run the performance test suite" +) +parser.add_argument( + "--destructive", action="store_true", help="Run the destructive test suite" +) +parser.add_argument( + "--thirdparty", action="store_true", help="Run the third-party test suite" +) +parser.add_argument( + "--secureboot", action="store_true", help="Check the Secure Boot signature" +) +parser.add_argument("--disable-retest", help="Disable re-testing.") +args = parser.parse_args() + + +if args.logfile: + with open(args.logfile) as fd: + results = json.load(fd) + try: + upload_results(results, args.anonymous) + sys.exit(0) + except Exception: + sys.exit(1) + +encoding = sys.getdefaultencoding() + +result = subprocess.run(["uname", "-r"], encoding=encoding, capture_output=True) +rpm_version = result.stdout.strip() +kernel_version, release_and_arch = rpm_version.split("-") +build_release, arch = release_and_arch.rsplit(".", 1) +with open("/etc/os-release") as fd: + for line in fd.readlines(): + if line.startswith("VERSION_ID="): + fedora_version = int(line.split("=")[1]) + break + else: + raise Exception("No VERSION_ID found in /etc/os-release") + +test_results = { + "kernel_version": kernel_version, + "build_release": build_release, + "arch": arch, + "fedora_version": fedora_version, + "tests": [], +} + + +directories = ["minimal", "default"] +if args.minimal: + directories = ["minimal"] +if args.destructive: + directories.append("destructive") +if args.performance: + directories.append("performance") +if args.secureboot: + directories.append("secureboot") +if args.stress: + directories.append("stress") +if args.thirdparty: + directories.append("thirdparty") +if args.stress: + directories.append("stress") + +# run test, check return code +# 0 = pass +# 3 = skip +# 4 - warn +# anything else is fail +root_dir = os.getcwd() +for directory in directories: + for dirpath, dirnames, filenames in os.walk(directory): + if "runtest.sh" in filenames: + os.chdir(dirpath) + print("Running " + dirpath) + result = subprocess.run( + ["./runtest.sh"], encoding=encoding, shell=True, capture_output=True + ) + os.chdir(root_dir) + test_results["tests"].append( + { + "name": dirpath, + "passed": result.returncode == 0, + "waived": result.returncode == 3, + "details": result.stdout, + } + ) + +logdir = os.path.join(root_dir, "logs", str(fedora_version), rpm_version) +logfile = os.path.join( + logdir, "{suite}-{date}.log".format(suite="-".join(directories), date=time.time()) +) + +os.makedirs(logdir, exist_ok=True) +with open(logfile, "w") as fd: + json.dump(test_results, fd, sort_keys=True, indent=4) + +if args.upload: + try: + upload_results(test_results, args.anonymous) + except Exception as e: + print("Failed to upload results: " + str(e)) + +print("Your log file is located at: " + logfile) +print("Submit your results to: https://apps.fedoraproject.org/kerneltest/") + +print("The following information is not submitted with your log;") +print("it is for informational purposes only.") + +if os.path.exists("/usr/bin/pesign"): + sign_result = subprocess.run( + ["pesign", "-i", "/boot/vmlinuz-{}".format(rpm_version), "-S"] + ) + print(sign_result) + +print("Vulnerability status:") +vulns = "/sys/devices/system/cpu/vulnerabilities/" +for vuln in os.listdir(vulns): + with open(os.path.join(vulns, vuln)) as fd: + print("{}: {}".format(vuln, fd.read().strip())) + + +if all([test["passed"] for test in test_results["tests"]]): + sys.exit(0) +else: + sys.exit(1) diff --git a/runtests.sh b/runtests.sh deleted file mode 100755 index d76e543..0000000 --- a/runtests.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash -# -# Licensed under the terms of the GNU GPL License version 2. - -date=$(date +%s) -logdir=$(pwd)/logs -logfile=$logdir/kernel-test-$date.log.txt -verbose=n -testset=default -cleanrun=PASS -failedtests=None -warntests=None -commit=n -commithook=/usr/bin/true -disable_retest=n -thirdparty=n - -if [ -f ./.config ]; then - source ./.config -else - echo "No .config file found. You can cp config.example .config and edit as needed for easier log submission." -fi - -kver=$(uname -r) -release=$(cat /etc/redhat-release) - -# Check for pre-requisites. -if [ ! -f /usr/bin/gcc ]; then - echo " Could not locate gcc. The Fedora kernel test suite requires gcc." - exit -fi - -# Unset MALLOC_CHECK_ and MALLOC_PERTURB_. Some tests might not work -# well with those active (e.g. libhugetlbfs). -unset MALLOC_CHECK_ -unset MALLOC_PERTURB_ - -if [ ! -d "$logdir" ] ; then - mkdir $logdir -fi - -args=y - -while [ $args = y ] -do - case "$1" in - -v) - #TO DO: Implement verbose behavior. - verbose=y - shift 1 - ;; - -t) - testset=$2 - shift 2 - ;; - *) - args=n - esac -done - - -if [ "$disable_retest" == "y" ]; then - # Check if wanted test has been executed with current kernel - # version. - for file in $(find $logdir -name \*.log.txt); do - version_tested=$(cat $file | sed -n 3p | cut -d ' ' -f 2) - test_executed=$(cat $file | sed -n 2p | cut -d ' ' -f 3) - - if [ "$version_tested" == "$kver" -a "$test_executed" == "$testset" ]; then - echo "The current kernel was already tested with this test, abort." - exit 0 - fi - done -fi - -case $testset in -minimal) - dirlist="minimal" - ;; -default) - dirlist="minimal default" - ;; -stress) - dirlist="minimal default stress" - ;; -destructive) - echo "You have specified the destructive test set." - echo "This test may cause damage to your system." - echo "Are you sure that you wish to continue?" - read continue - if [ $continue == 'y' ] ; then - dirlist="minimal default destructive" - else - dirlist="minimal default" - testset=default - fi - ;; -performance) - dirlist="performance" - ;; -*) - echo "Supported test sets are minimal, default, stress, destructive or performance." - exit 1 -esac - -# Test Secure Boot? -if [ "$checksig" == "y" ]; then - dirlist="secureboot $dirlist" -fi - -# Test Third Party Modules? -if [ "$thirdparty" == "y" ]; then - dirlist="$dirlist thirdparty" -fi - -# Basic logfile headers. -echo "Date: $(date)" > $logfile -echo "Test set: $testset" >> $logfile -echo "Kernel: $kver" >> $logfile -echo "Release: $release" >> $logfile -echo "Result: RESULTHOLDER" >> $logfile -echo "============================================================" >>$logfile - - - -# Start running tests. -echo "Test suite called with $testset" - -for dir in $dirlist -do - for test in $(find ./$dir/ -name runtest.sh) - do - testdir=$(dirname $test) - pushd $testdir &>/dev/null - #TO DO: Purpose file test name format. - testname=$testdir - echo "Starting test $testname" >> $logfile - - if [ "$testset" == "performance" ]; then - ./runtest.sh >>$logfile - elif [ "$dir" == "secureboot" ]; then - ./runtest.sh "$validsig" &>>$logfile - else - ./runtest.sh &>>$logfile - fi - complete=$? - case $complete in - 0) - result=PASS - ;; - 3) - result=SKIP - ;; - 4) - result=WARN - ;; - *) - result=FAIL - esac - printf "%-65s%-8s\n" "$testname" "$result" - if [ "$result" == "FAIL" ]; then - cleanrun=FAIL - if [ "$failedtests" == "None" ]; then - failedtests="$testname" - else - failedtests="$failedtests $testname" - fi - fi - if [ "$result" == "WARN" ]; then - if [ "$cleanrun" == "PASS" ]; then - cleanrun=WARN - fi - if [ "$warntests" == "None" ]; then - warntests="$testname" - else - warntests="$warntests $testname" - fi - fi - popd &>/dev/null - done -done - -# Fix up logfile headers. -sed -i "s,RESULTHOLDER,$cleanrun\nFailed Tests: $failedtests\nWarned Tests: $warntests,g" $logfile -printf "\n%-65s%-8s\n" "Test suite complete" "$cleanrun" - -if [ "$commit" == "y" ]; then - printf "\nYour log file is being submitted...\n" - $commithook -else - printf "\nYour log file is located at: $logfile\n" - printf "Submit your results to: https://apps.fedoraproject.org/kerneltest/\n" -fi - -echo "The following information is not submitted with your log;" -echo "it is for informational purposes only." - -if [ -f /usr/bin/pesign ]; then - echo "Checking for kernel signature:" - /usr/bin/pesign -i /boot/vmlinuz-$kver -S | grep "common name" -fi - -echo "Vulnerability status:" -grep . /sys/devices/system/cpu/vulnerabilities/* - -if [ "$cleanrun" == "FAIL" ]; then - exit 1 -else - exit 0 -fi -