#294 Add coverage measurement to the test suite
Opened 6 years ago by puiterwijk. Modified 6 years ago
puiterwijk/ipsilon coverage  into  master

file modified
+5 -1
@@ -170,7 +170,11 @@ 

  

  containertest-fedora27: container-fedora27

  	@echo "Starting Fedora 27 tests ..."

- 	@docker run -v `pwd`:/code -t --rm ipsilon-fedora27

+ 	# This one is special, since it's the latest version, so supposed to support the most features.

+ 	# As such, we do coverage testing on this one.

+ 	@docker run -v `pwd`:/code -t --rm --entrypoint /usr/bin/make ipsilon-fedora27 testdeps lp-test unittests

+ 	# This "coverage-min" number should be increased everytime the coverage increases to make sure we don't regress.

+ 	@docker run -v `pwd`:/code -t --rm --entrypoint /usr/bin/bash ipsilon-fedora27 ./runtests --path=$(TESTDIR) --coverage --coverage-min=68

  	@echo "Fedora 27 passed"

  

  containertest-lint: container-centos7

@@ -95,6 +95,11 @@ 

          shutil.move(idp_conf, '%s.backup.%s' % (idp_conf, now))

      if not os.path.exists(instance_conf):

          os.makedirs(instance_conf, 0700)

+     coverage = ''

+     if args['coverage'] or args['branch_coverage']:

+         coverage += 'coverage.enabled = True'

+         if args['coverage_branch']:

+             coverage += '\ncoverage.branch = True'

      confopts = {'instance': args['instance'],

                  'instanceurl': args['instanceurl'],

                  'needs_mount': args.get('needs_mount'),
@@ -118,7 +123,8 @@ 

                      'database_url'] % {'datadir': args['data_dir'],

                                         'dbname': 'saml2sessions'},

                  'secure': "False" if args['secure'] == "no" else "True",

-                 'debugging': "True" if args['server_debugging'] else "False"}

+                 'debugging': "True" if args['server_debugging'] else "False",

+                 'coverage': coverage}

      # Testing database sessions

      if 'session_type' in args:

          confopts['sesstype'] = args['session_type']
@@ -424,6 +430,10 @@ 

      parser.add_argument('--session-timeout', default=30, type=int,

                          help='Time that sessions are valid for (in minutes, ' +

                               'default: 30 minutes)')

+     parser.add_argument('--coverage', default=False, action='store_true',

+                         help=argparse.SUPPRESS)

+     parser.add_argument('--coverage-branch', default=False,

+                         action='store_true', help=argparse.SUPPRESS)

  

      lms = []

      azs = []

file modified
+19
@@ -15,6 +15,12 @@ 

  import atexit

  import string

  import cherrypy

+ 

+ try:

+     import coverage

+ except ImportError:  # pragma: no cover

+     coverage = None

+ 

  from ipsilon import find_config

  from ipsilon.util.data import AdminStore

  from ipsilon.util import page
@@ -41,6 +47,19 @@ 

  cherrypy.lib.sessions.EtcdSession = ipsilon.util.sessions.EtcdSession

  cherrypy.config.update(cfgfile)

  

+ # If requested, let's capture coverage data

+ if cherrypy.config.get('coverage.enabled', False):

+     if not coverage:  # pragma: no cover

+         raise Exception('Coverage.py was unavailable')

+ 

+     cov = coverage.Coverage(

+         data_file='coverage',

+         data_suffix=True,

+         auto_data=True,

+         branch=cherrypy.config.get('coverage.branch', False))

+     cov.start()

+     atexit.register(cov.stop)

+ 

  # Force cherrypy logging to work. Note that this ignores the config-file

  # setting.

  cherrypy.log.screen = True

@@ -21,3 +21,5 @@ 

  tools.sessions.timeout = ${session_timeout}

  tools.sessions.httponly = ${secure}

  tools.sessions.secure = ${secure}

+ 

+ ${coverage}

@@ -1,1 +1,1 @@ 

- RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib

+ RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib python2-coverage

file modified
+27 -1
@@ -9,6 +9,7 @@ 

  import shutil

  import signal

  import random

+ import time

  from string import Template

  import subprocess

  
@@ -87,6 +88,8 @@ 

          self.print_cases = False

          self.stdout = None

          self.stderr = None

+         self.coverage = False

+         self.coverage_branch = False

  

      def platform_supported(self):

          """This return whether the current platform supports this test.
@@ -164,6 +167,10 @@ 

          newconf.add_section('arguments')

          for k in args_opts:

              newconf.set('arguments', k, args_opts[k])

+         if self.coverage:

+             newconf.set('arguments', 'coverage', True)

+         if self.coverage_branch:

+             newconf.set('arguments', 'coverage_branch', True)

  

          profile = io.BytesIO()

          newconf.write(profile)
@@ -246,7 +253,13 @@ 

          cmd = [os.path.join(self.rootdir,

                              'ipsilon/install/ipsilon-server-install'),

                 '--config-profile=%s' % profile]

-         subprocess.check_call(cmd, env=env,

+         if self.coverage:

+             env['COVERAGE_FILE'] = 'installer.coverage'

+             covcmd = ['coverage', 'run']

+             if self.coverage_branch:

+                 covcmd.append('--branch')

+             cmd = covcmd + cmd

+         subprocess.check_call(cmd, env=env, cwd=self.testdir,

                                stdout=self.stdout, stderr=self.stderr)

          os.symlink(os.path.join(self.rootdir, 'ipsilon'),

                     os.path.join(self.testdir, 'lib', name, 'ipsilon'))
@@ -435,9 +448,22 @@ 

              if ksetup.returncode != 0:

                  raise ValueError('kinit %s failed' % self.testuser)

  

+     def wait_process(self, p):

+         start = int(time.time())

+         while True:

+             if int(time.time()) - start > 5:

+                 print('Process %s did not terminate in time, killing' % p.pid)

+                 p.kill()

+                 return

+             if p.poll() is not None:

+                 return

+             time.sleep(0.1)

+ 

      def wait(self):

          for p in self.processes:

              os.killpg(p.pid, signal.SIGTERM)

+         for p in self.processes:

+             self.wait_process(p)

  

      def setup_servers(self, env=None):

          raise NotImplementedError()

file modified
+58
@@ -9,6 +9,7 @@ 

  

  import argparse

  from ipsilon.util import plugin

+ import glob

  import os

  import sys

  import subprocess
@@ -43,6 +44,13 @@ 

                          help='Test results header')

      parser.add_argument('--path', default='%s/testdir' % os.getcwd(),

                          help="Directory in which tests are run")

+     parser.add_argument('--coverage', default=False, action='store_true',

+                         help="Whether or not to collect coverage reports")

+     parser.add_argument('--coverage-branch', default=False,

+                         action='store_true',

+                         help='Whether to do branch coverage')

+     parser.add_argument('--coverage-min', default=0,

+                         help='Minimum coverage % to consider success')

      parser.add_argument('--fail-on-first-error', '-x', action='store_true',

                          help='Abort test run on first test failure')

      parser.add_argument('--test', action='append', default=None,
@@ -105,6 +113,8 @@ 

          devnull = open(os.devnull, 'w')

          test.stdout = devnull

          test.stderr = devnull

+     test.coverage = args['coverage']

+     test.coverage_branch = args['coverage_branch']

  

      if args['verbose'] >= VERBOSE_SHOWCASES:

          test.print_cases = True
@@ -158,6 +168,22 @@ 

  def main():

      args = parse_args()

  

+     if args['coverage_min'] or args['coverage_branch']:

+         args['coverage_min'] = int(args['coverage_min'])

+         args['coverage'] = True

+ 

+     if args['coverage']:

+         import coverage

+         cov = coverage.Coverage(

+             data_file=os.path.join(args['path'], 'tests.coverage'),

+             data_suffix=False,

+             auto_data=False,

+             branch=args['coverage_branch']

+         )

+         cov.start()

+ 

+     coverage_files = [os.path.join(args['path'], 'tests.coverage')]

+ 

      tests = get_tests()

      if args['list_tests']:

          for testname in tests.keys():
@@ -179,6 +205,7 @@ 

          os.makedirs(args['path'])

  

      test_results = {}

+     anyfailures = False

  

      for test in args['test']:

          if args['verbose'] >= VERBOSE_SHOWTESTS:
@@ -189,9 +216,34 @@ 

          if args['verbose'] >= VERBOSE_SHOWTESTS:

              print(result_to_str(result))

  

+         anyfailures = anyfailures or result_is_fail(result)

+ 

          if args['fail_on_first_error'] and result_is_fail(result):

              break

  

+         coverage_files.append(os.path.join(args['path'],

+                               test,

+                               'installer.coverage'))

+         coverage_files.extend(glob.glob(

+             '%s/%s/lib/*/coverage.*' % (args['path'], test)))

+ 

+     if args['coverage']:

+         cov.stop()

+         cov.save()

+ 

+     if args['coverage'] and not anyfailures:

+         print('Collecting coverage reports...')

+ 

+         cov = coverage.Coverage(

+             data_file=os.path.join(args['path'], 'combined.coverage'),

+             branch=args['coverage_branch'])

+         cov.combine(data_paths=coverage_files)

+         cov.save()

+         cov_perc = cov.report(include='ipsilon/*')

+         cov.xml_report(

+             include='ipsilon/*',

+             outfile=os.path.join(args['path'], 'combined.coverage.xml'))

+ 

      if not args['no_overview']:

          print(args['results_header'])

          for test in test_results:
@@ -201,6 +253,12 @@ 

             for result in test_results.values()):

          sys.exit(1)

  

+     if (not anyfailures and args['coverage_min'] and

+             cov_perc < args['coverage_min']):

+         print('Test coverage was %d%%, less than required %d%%' %

+               (cov_perc, args['coverage_min']))

+         sys.exit(1)

+ 

  

  if __name__ == '__main__':

      main()

This also makes the Fedora27 containertest run the coverage analysis, with a minimum percentage coverage.

rebased onto 4cced0586705f97ff37527fea0231b350d873d37

6 years ago

s/to most/the most/

For PEP-8, I recommend moving this import to the top of the file.

rebased onto ff5bfcbdd97b2ac131b00d9989a0100da3b3de69

6 years ago

rebased onto 4320ec4

6 years ago

Not sure if this is still relevant when Fedora 27 is no longer supported, but otherwise LGTM