| |
@@ -9,6 +9,7 @@
|
| |
|
| |
import argparse
|
| |
from ipsilon.util import plugin
|
| |
+ import glob
|
| |
import os
|
| |
import sys
|
| |
import subprocess
|
| |
@@ -43,6 +44,13 @@
|
| |
help='Test results header')
|
| |
parser.add_argument('--path', default='%s/testdir' % os.getcwd(),
|
| |
help="Directory in which tests are run")
|
| |
+ parser.add_argument('--coverage', default=False, action='store_true',
|
| |
+ help="Whether or not to collect coverage reports")
|
| |
+ parser.add_argument('--coverage-branch', default=False,
|
| |
+ action='store_true',
|
| |
+ help='Whether to do branch coverage')
|
| |
+ parser.add_argument('--coverage-min', default=0,
|
| |
+ help='Minimum coverage % to consider success')
|
| |
parser.add_argument('--fail-on-first-error', '-x', action='store_true',
|
| |
help='Abort test run on first test failure')
|
| |
parser.add_argument('--test', action='append', default=None,
|
| |
@@ -105,6 +113,8 @@
|
| |
devnull = open(os.devnull, 'w')
|
| |
test.stdout = devnull
|
| |
test.stderr = devnull
|
| |
+ test.coverage = args['coverage']
|
| |
+ test.coverage_branch = args['coverage_branch']
|
| |
|
| |
if args['verbose'] >= VERBOSE_SHOWCASES:
|
| |
test.print_cases = True
|
| |
@@ -158,6 +168,22 @@
|
| |
def main():
|
| |
args = parse_args()
|
| |
|
| |
+ if args['coverage_min'] or args['coverage_branch']:
|
| |
+ args['coverage_min'] = int(args['coverage_min'])
|
| |
+ args['coverage'] = True
|
| |
+
|
| |
+ if args['coverage']:
|
| |
+ import coverage
|
| |
+ cov = coverage.Coverage(
|
| |
+ data_file=os.path.join(args['path'], 'tests.coverage'),
|
| |
+ data_suffix=False,
|
| |
+ auto_data=False,
|
| |
+ branch=args['coverage_branch']
|
| |
+ )
|
| |
+ cov.start()
|
| |
+
|
| |
+ coverage_files = [os.path.join(args['path'], 'tests.coverage')]
|
| |
+
|
| |
tests = get_tests()
|
| |
if args['list_tests']:
|
| |
for testname in tests.keys():
|
| |
@@ -179,6 +205,7 @@
|
| |
os.makedirs(args['path'])
|
| |
|
| |
test_results = {}
|
| |
+ anyfailures = False
|
| |
|
| |
for test in args['test']:
|
| |
if args['verbose'] >= VERBOSE_SHOWTESTS:
|
| |
@@ -189,9 +216,34 @@
|
| |
if args['verbose'] >= VERBOSE_SHOWTESTS:
|
| |
print(result_to_str(result))
|
| |
|
| |
+ anyfailures = anyfailures or result_is_fail(result)
|
| |
+
|
| |
if args['fail_on_first_error'] and result_is_fail(result):
|
| |
break
|
| |
|
| |
+ coverage_files.append(os.path.join(args['path'],
|
| |
+ test,
|
| |
+ 'installer.coverage'))
|
| |
+ coverage_files.extend(glob.glob(
|
| |
+ '%s/%s/lib/*/coverage.*' % (args['path'], test)))
|
| |
+
|
| |
+ if args['coverage']:
|
| |
+ cov.stop()
|
| |
+ cov.save()
|
| |
+
|
| |
+ if args['coverage'] and not anyfailures:
|
| |
+ print('Collecting coverage reports...')
|
| |
+
|
| |
+ cov = coverage.Coverage(
|
| |
+ data_file=os.path.join(args['path'], 'combined.coverage'),
|
| |
+ branch=args['coverage_branch'])
|
| |
+ cov.combine(data_paths=coverage_files)
|
| |
+ cov.save()
|
| |
+ cov_perc = cov.report(include='ipsilon/*')
|
| |
+ cov.xml_report(
|
| |
+ include='ipsilon/*',
|
| |
+ outfile=os.path.join(args['path'], 'combined.coverage.xml'))
|
| |
+
|
| |
if not args['no_overview']:
|
| |
print(args['results_header'])
|
| |
for test in test_results:
|
| |
@@ -201,6 +253,12 @@
|
| |
for result in test_results.values()):
|
| |
sys.exit(1)
|
| |
|
| |
+ if (not anyfailures and args['coverage_min'] and
|
| |
+ cov_perc < args['coverage_min']):
|
| |
+ print('Test coverage was %d%%, less than required %d%%' %
|
| |
+ (cov_perc, args['coverage_min']))
|
| |
+ sys.exit(1)
|
| |
+
|
| |
|
| |
if __name__ == '__main__':
|
| |
main()
|
| |
This also makes the Fedora27 containertest run the coverage analysis, with a minimum percentage coverage.