#199 Debugger enablement for testsuites
Merged 6 years ago by cipherboy. Opened 6 years ago by cipherboy.
cipherboy/gssproxy debug_enablement  into  master

file modified
+1 -1
@@ -317,7 +317,7 @@ 

      systemd/gssproxy.service

  

  check: all $(check_PROGRAMS)

- 	$(srcdir)/tests/runtests.py

+ 	$(srcdir)/tests/runtests.py $(CHECKARGS)

  

  tests: check

  

file modified
+40 -13
@@ -7,16 +7,37 @@ 

  import sys

  import traceback

  

+ import testlib

  from testlib import *

  

  def parse_args():

      parser = argparse.ArgumentParser(description='GSS-Proxy Tests Environment')

      parser.add_argument('--path', default='%s/testdir' % os.getcwd(),

                          help="Directory in which tests are run")

- 

-     return vars(parser.parse_args())

- 

- if __name__ == '__main__':

+     parser.add_argument('--debug-all', default=False, action="store_true",

+                         help="Enable debugging for all test cases")

+     parser.add_argument('--debug-gssproxy', default=False, action="store_true",

+                         help="Enable debugging for gssproxy command")

+     parser.add_argument('--debug-cmd', default="gdb --args",

+                         help="Set the debugging command. Defaults to gdb " +

+                               "--args")

+     parser.add_argument('--debug-num', default=-1, type=int,

+                         help="Specify the testcase number to debug")

+     parser.add_argument('--timeout', default=15, type=int,

+                         help="Specify test case timeout limit")

+     parser.add_argument('--valgrind-cmd', default="valgrind " +

+                         "--track-origins=yes",

+                         help="Set the valgrind command. Defaults to " +

+                         "valgrind --track-origins=yes")

+     parser.add_argument('--force-valgrind', default=False, action="store_true",

+                         help="Force valgrind to be run on all test cases")

+ 

+     args = vars(parser.parse_args())

+     testlib_process_args(args)

+ 

+     return args

+ 

+ def runtests_main(testfiles):

      args = parse_args()

  

      testdir = args['path']
@@ -41,6 +62,7 @@ 

          keysenv = setup_keys(testdir, kdcenv)

  

          gssapienv = setup_gssapi_env(testdir, kdcenv)

+         gssapienv['TERM'] = os.environ['TERM']

  

          gssproxylog = os.path.join(testdir, 'gssproxy.log')

  
@@ -59,23 +81,16 @@ 

          basicconf["gpid"] = gproc.pid

          basicconf["keysenv"] = keysenv

  

-         testnum = 0

-         testfiles = [f for f in os.listdir(os.path.dirname(sys.argv[0])) \

-                      if f.endswith(".py") and f.startswith("t_")]

-         testfiles.sort()

          print("Tests to be run: " + ", ".join(testfiles))

          for f in testfiles:

              fmod = f[:-len(".py")]

              t = importlib.__import__(fmod)

  

-             basicconf['prefix'] = '%02d' % testnum

-             logfile = os.path.join(testdir, "%02d_%s.log" % (testnum, fmod))

-             basicconf['logfile'] = open(logfile, 'a')

+             basicconf['prefix'] = str(testlib.cmd_index)

+             basicconf['logpath'] = testdir

              r = t.run(testdir, gssapienv, basicconf)

              if r != 0:

                  errored = True

- 

-             testnum += 1

      except Exception:

          traceback.print_exc()

          errored = True
@@ -87,3 +102,15 @@ 

          if errored:

              sys.exit(1)

          sys.exit(0)

+ 

+ if __name__ == "__main__":

+     print("\n")

+     print("To pass arguments to the test suite, use CHECKARGS:")

+     print("    make check CHECKARGS='--debug-num=<num>'")

+     print("A full set of available options can be seen with --help")

+     print("\n")

+ 

+     testfiles = [f for f in os.listdir(os.path.dirname(sys.argv[0])) \

+                  if f.endswith(".py") and f.startswith("t_")]

+     testfiles.sort()

+     runtests_main(testfiles)

file modified
+7 -13
@@ -5,7 +5,7 @@ 

  

  def run(testdir, env, conf, expected_failure=False):

      print("Testing basic acquire creds...", file=sys.stderr)

-     logfile = conf['logfile']

+     conf['prefix'] = str(cmd_index)

  

      svc_keytab = os.path.join(testdir, SVC_KTNAME)

      testenv = {'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +
@@ -17,16 +17,10 @@ 

                 'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'}

      testenv.update(env)

  

-     cmd = ["./tests/t_acquire", conf['svc_name']]

-     print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, env), file=logfile)

-     logfile.flush()

+     cmd = "./tests/t_acquire " + conf['svc_name']

  

-     p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,

-                           env=testenv, preexec_fn=os.setsid)

-     try:

-         p1.wait(10)

-     except subprocess.TimeoutExpired:

-         # p1.returncode is set to None here

-         pass

-     print_return(p1.returncode, "Acquire", expected_failure)

-     return p1.returncode if not expected_failure else int(not p1.returncode)

+     return run_testcase_cmd(testenv, conf, cmd, "Acquire", expected_failure)

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_acquire.py"])

file modified
+100 -13
@@ -1,11 +1,24 @@ 

  #!/usr/bin/python3

  # Copyright (C) 2014,2015,2016 - GSS-Proxy contributors; see COPYING for the license

  

- from testlib import *

+ import testlib

+ import os

+ import signal

+ import subprocess

+ import sys

+ 

  

  def run(testdir, env, conf, expected_failure=False):

      print("Testing basic init/accept context", file=sys.stderr)

-     logfile = conf['logfile']

+     conf['prefix'] = str(testlib.cmd_index)

+ 

+     init_logfile = os.path.join(conf['logpath'], "test_%d.log" %

+                                 testlib.cmd_index)

+     init_logfile = open(init_logfile, 'a')

+ 

+     accept_logfile = os.path.join(conf['logpath'], "test_%d.log" %

+                                   (testlib.cmd_index + 1))

+     accept_logfile = open(accept_logfile, 'a')

  

      svcenv = {'KRB5_KTNAME': conf['keytab'],

                'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +
@@ -20,6 +33,9 @@ 

      else:

          init_cmd = ["./tests/t_init", conf['svc_name']]

  

+     init_cmd = " ".join(init_cmd)

+     accept_cmd = " ".join(["./tests/t_accept"])

+ 

      clienv = {'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +

                                                    '_init.ccache'),

                'KRB5_TRACE': os.path.join(testdir, 't' + conf['prefix'] +
@@ -28,27 +44,94 @@ 

                'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'}

      clienv.update(env)

  

-     print("[SVCENV]\n%s\n[CLIENV]\n%s\nCLI NAME: %s\n" % (

-           svcenv, clienv, client_name), file=logfile)

+     print("[CLIENV]\n%s\nCLI NAME: %s\n" % (

+           clienv, client_name), file=init_logfile)

+ 

+     print("[SRVENV]\n%s\n" % (svcenv), file=accept_logfile)

+ 

+     init_logfile.flush()

+     accept_logfile.flush()

  

      pipe0 = os.pipe()

      pipe1 = os.pipe()

  

+     if testlib.debug_cmd_index == testlib.cmd_index:

+         p1 = subprocess.Popen(init_cmd,

+                               stdin=pipe0[0], stdout=pipe1[1],

+                               stderr=init_logfile, env=clienv,

+                               preexec_fn=os.setsid, shell=True,

+                               executable="/bin/bash")

+         print("PID: %d\n" % p1.pid)

+         print("Attach and start debugging, then press enter to start t_init.")

+         input()

+ 

+         p2 = subprocess.Popen(["./tests/t_accept"],

+                               stdin=pipe1[0], stdout=pipe0[1],

+                               stderr=accept_logfile, env=svcenv,

+                               preexec_fn=os.setsid, shell=True,

+                               executable="/bin/bash")

+         print("To resume tests if hung, kill pid %d\n" % p2.pid)

+         p2.wait()

+ 

+         init_logfile.close()

+         accept_logfile.close()

+ 

+         testlib.cmd_index += 2

+         return int(expected_failure)

+     elif testlib.debug_cmd_index == testlib.cmd_index+1:

+         p2 = subprocess.Popen(["./tests/t_accept"],

+                               stdin=pipe1[0], stdout=pipe0[1],

+                               stderr=accept_logfile, env=svcenv,

+                               preexec_fn=os.setsid, shell=True,

+                               executable="/bin/bash")

+         print("PID: %d\n" % p2.pid)

+         print("Attach and start debugging, then press enter to start t_init.")

+         input()

+ 

+         p1 = subprocess.Popen(init_cmd,

+                               stdin=pipe0[0], stdout=pipe1[1],

+                               stderr=init_logfile, env=clienv,

+                               preexec_fn=os.setsid, shell=True,

+                               executable="/bin/bash")

+         print("To resume tests if hung, kill pid %d\n" % p1.pid)

+         p1.wait()

+ 

+         init_logfile.close()

+         accept_logfile.close()

+ 

+         testlib.cmd_index += 2

+         return int(expected_failure)

+ 

+     if testlib.valgrind_everywhere:

+         accept_cmd = testlib.valgrind_cmd + accept_cmd

+         init_cmd = testlib.valgrind_cmd + init_cmd

+     p2 = subprocess.Popen(accept_cmd,

+                           stdin=pipe1[0], stdout=pipe0[1],

+                           stderr=accept_logfile, env=svcenv,

+                           preexec_fn=os.setsid, shell=True,

+                           executable="/bin/bash")

      p1 = subprocess.Popen(init_cmd,

                            stdin=pipe0[0], stdout=pipe1[1],

-                           stderr=logfile, env=clienv, preexec_fn=os.setsid)

-     p2 = subprocess.Popen(["./tests/t_accept"],

-                           stdin=pipe1[0], stdout=pipe0[1],

-                           stderr=logfile, env=svcenv, preexec_fn=os.setsid)

+                           stderr=init_logfile, env=clienv,

+                           preexec_fn=os.setsid, shell=True,

+                           executable="/bin/bash")

  

      try:

-         p1.wait(10)

-         p2.wait(10)

+         p1.wait(testlib.testcase_wait)

+         p2.wait(testlib.testcase_wait)

      except subprocess.TimeoutExpired:

          # {p1,p2}.returncode are set to None here

-         pass

-     print_return(p1.returncode, "Init", expected_failure)

-     print_return(p2.returncode, "Accept", expected_failure)

+         if not expected_failure:

+             testlib.print_warning("warning", "timeout")

+     init_logfile.close()

+     accept_logfile.close()

+     testlib.print_return(p1.returncode, testlib.cmd_index,

+                          "(%d) Init" % testlib.cmd_index,

+                          expected_failure)

+     testlib.print_return(p2.returncode, testlib.cmd_index + 1,

+                          "(%d) Accept" % (testlib.cmd_index + 1),

+                          expected_failure)

+     testlib.cmd_index += 2

      try:

          os.killpg(p1.pid, signal.SIGTERM)

          os.killpg(p2.pid, signal.SIGTERM)
@@ -59,3 +142,7 @@ 

      elif p2.returncode != 0:

          return p2.returncode if not expected_failure else int(not p2.returncode)

      return int(expected_failure)

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_basic.py"])

file modified
+10 -13
@@ -5,7 +5,10 @@ 

  

  def run(testdir, env, conf):

      print("Testing cred store extensions...", file=sys.stderr)

-     logfile = conf["logfile"]

+     conf['prefix'] = str(cmd_index)

+ 

+     logfile = os.path.join(conf["logpath"], "test_%d.log" % cmd_index)

+     logfile = open(logfile, 'a')

  

      ccache = "FILE:" + os.path.join(testdir, "t" + conf["prefix"] +

                                      "_cred_store.ccache")
@@ -26,16 +29,10 @@ 

      testenv.update(env)

      temp_ccache = "FILE:" + os.path.join(testdir, "t" + conf["prefix"] +

                                           "_temp.ccache")

-     cmd = ["./tests/t_cred_store", ccache, temp_ccache]

-     print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, testenv), file=logfile)

-     logfile.flush()

+     cmd = " ".join(["./tests/t_cred_store", ccache, temp_ccache])

+ 

+     return run_testcase_cmd(testenv, conf, cmd, "Cred store")

  

-     p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,

-                           env=testenv, preexec_fn=os.setsid)

-     try:

-         p1.wait()

-     except subprocess.TimeoutExpired:

-         # p1.returncode is set to None here

-         pass

-     print_return(p1.returncode, "Cred store", False)

-     return p1.returncode

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_cred_store.py"])

file modified
+20 -29
@@ -35,8 +35,7 @@ 

  '''

  

  def run_cmd(testdir, env, conf, name, socket, cmd, keytab, expected_failure):

- 

-     logfile = conf['logfile']

+     conf['prefix'] = str(cmd_index)

      testenv = env.copy()

      testenv.update({'KRB5CCNAME': os.path.join(testdir, 't' + conf['prefix'] +

                                                 '_impersonate.ccache'),
@@ -47,19 +46,7 @@ 

                      'GSSPROXY_SOCKET': socket,

                      'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'})

  

-     print("\nTesting: [%s]" % (name,), file=logfile)

-     print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, testenv), file=logfile)

-     logfile.flush()

- 

-     p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,

-                           env=testenv, preexec_fn=os.setsid)

-     try:

-         p1.wait(10)

-     except subprocess.TimeoutExpired:

-         # p1.returncode is set to None here

-         pass

-     print_return(p1.returncode, name, expected_failure)

-     return p1.returncode if not expected_failure else int(not p1.returncode)

+     return run_testcase_cmd(testenv, conf, cmd, name, expected_failure)

  

  def run(testdir, env, conf):

      print("Testing impersonate creds...", file=sys.stderr)
@@ -77,56 +64,56 @@ 

      # Test all permitted

      msg = "Impersonate"

      socket = os.path.join(testdir, 'impersonate.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

-            path_prefix + 'impersonate.cache']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

+            path_prefix + 'impersonate.cache'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, False)

      rets.append(r)

  

      #Test self fail

      msg = "Impersonate fail self"

      socket = os.path.join(testdir, 'impersonate-proxyonly.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

-            path_prefix + 'impersonate.cache']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

+            path_prefix + 'impersonate.cache'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, True)

      rets.append(r)

  

      #Test proxy fail

      msg = "Impersonate fail proxy"

      socket = os.path.join(testdir, 'impersonate-selfonly.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

-            path_prefix + 'impersonate.cache']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

+            path_prefix + 'impersonate.cache'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, True)

      rets.append(r)

  

      #Test s4u2self half succeed

      msg = "s4u2self delegation"

      socket = os.path.join(testdir, 'impersonate-selfonly.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

-            path_prefix + 'impersonate.cache', 's4u2self']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

+            path_prefix + 'impersonate.cache', 's4u2self'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, False)

      rets.append(r)

  

      #Test proxy to self succeed

      msg = "Impersonate to self"

      socket = os.path.join(testdir, 'impersonate-selfonly.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, HOST_GSS,

-            path_prefix + 'impersonate.cache', 's4u2proxy']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, HOST_GSS,

+            path_prefix + 'impersonate.cache', 's4u2proxy'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, SVC_KTNAME, False)

      rets.append(r)

  

      #Test s4u2proxy half fail

      msg = "s4u2proxy fail"

      socket = os.path.join(testdir, 'impersonate-selfonly.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

-            path_prefix + 'impersonate.cache', 's4u2proxy']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

+            path_prefix + 'impersonate.cache', 's4u2proxy'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, True)

      rets.append(r)

  

      #Test s4u2proxy half succeed

      msg = "s4u2proxy"

      socket = os.path.join(testdir, 'impersonate-proxyonly.socket')

-     cmd = ["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

-            path_prefix + 'impersonate.cache', 's4u2proxy']

+     cmd = " ".join(["./tests/t_impersonate", USR_NAME, HOST_GSS, PROXY_GSS,

+            path_prefix + 'impersonate.cache', 's4u2proxy'])

      r = run_cmd(testdir, env, conf, msg, socket, cmd, PROXY_KTNAME, False)

      rets.append(r)

  
@@ -139,3 +126,7 @@ 

      if len(e) > 0:

          return e[0]

      return 0

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_impersonate.py"])

file modified
+9 -8
@@ -5,7 +5,9 @@ 

  

  def run(testdir, env, conf):

      print("Testing interposer...", file=sys.stderr)

-     logfile = conf['logfile']

+     conf['prefix'] = str(cmd_index)

+     logfile = os.path.join(conf['logpath'], "test_%d.log" % cmd_index)

+     logfile = open(logfile, 'a')

  

      ienv = {"KRB5CCNAME": os.path.join(testdir, 'interpose_ccache'),

              "KRB5_KTNAME": os.path.join(testdir, SVC_KTNAME)}
@@ -19,10 +21,9 @@ 

      if ksetup.returncode != 0:

          raise ValueError('Kinit %s failed' % USR_NAME)

  

-     itest = subprocess.Popen(["./interposetest", "-t",

-                               "host@%s" % WRAP_HOSTNAME],

-                              stdout=logfile, stderr=logfile,

-                              env=ienv)

-     itest.wait()

-     print_return(itest.returncode, "Interpose", False)

-     return itest.returncode

+     cmd = " ".join(["./interposetest", "-t", "host@%s" % WRAP_HOSTNAME])

+     return run_testcase_cmd(ienv, conf, cmd, "Interpose")

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_interpose.py"])

@@ -19,6 +19,7 @@ 

  

  def run(testdir, env, conf):

      setup_multi_keys(testdir, env)

+     conf['prefix'] = str(cmd_index)

      prefix = conf["prefix"]

  

      print("Testing multiple keys Keytab with first principal",
@@ -56,3 +57,7 @@ 

      elif r2 != 0:

          return r2

      return 0

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_multi_key.py"])

file modified
+6 -1
@@ -22,6 +22,7 @@ 

  '''

  

  def run(testdir, env, conf):

+     conf['prefix'] = str(cmd_index)

      prefix = conf["prefix"]

      retval = 0

  
@@ -47,5 +48,9 @@ 

      os.kill(conf["gpid"], signal.SIGHUP)

      time.sleep(1)

  

-     print_return(retval, "Program", False)

+     print_return(retval, -1, "Program", False)

      return retval

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_program.py"])

@@ -5,6 +5,7 @@ 

  from t_basic import run as run_basic_test

  

  def run(testdir, env, basicconf):

+     basicconf['prefix'] = str(cmd_index)

      prefix = basicconf['prefix']

      keysenv = basicconf["keysenv"]

  
@@ -56,3 +57,7 @@ 

      if len(e) > 0:

          return e[0]

      return 0

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_reloading.py"])

file modified
+10 -14
@@ -5,10 +5,13 @@ 

  

  def run(testdir, env, conf):

      print("Testing setting credential options...", file=sys.stderr)

+     conf['prefix'] = str(cmd_index)

      path_prefix = os.path.join(testdir, 't' + conf['prefix'] + '_')

      init_ccache = path_prefix + 'sco_init.ccache'

  

-     logfile = conf['logfile']

+     logfile = os.path.join(conf['logpath'], "test_%d.log" % cmd_index)

+     logfile = open(logfile, 'a')

+ 

      testenv = env.copy()

      testenv.update({'KRB5CCNAME': init_ccache})

  
@@ -21,22 +24,15 @@ 

          raise ValueError("Kinit %s failed" % USR_NAME)

  

  

-     cmd = ["./tests/t_setcredopt", USR_NAME, HOST_GSS, init_ccache]

+     cmd = " ".join(["./tests/t_setcredopt", USR_NAME, HOST_GSS, init_ccache])

  

      testenv.update({'KRB5CCNAME': path_prefix + 'sco.ccache',

                      'KRB5_KTNAME': os.path.join(testdir, PROXY_KTNAME),

                      'KRB5_TRACE': path_prefix + 'sco.trace',

                      'GSSPROXY_BEHAVIOR': 'REMOTE_FIRST'})

  

-     print("[COMMAND]\n%s\n[ENVIRONMENT]\n%s\n" % (cmd, testenv), file=logfile)

-     logfile.flush()

- 

-     p1 = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=logfile,

-                           env=testenv, preexec_fn=os.setsid)

-     try:

-         p1.wait(10)

-     except subprocess.TimeoutExpired:

-         # p1.returncode is set to None here

-         pass

-     print_return(p1.returncode, "Set cred options", False)

-     return p1.returncode

+     return run_testcase_cmd(testenv, conf, cmd, "Set cred options")

+ 

+ if __name__ == "__main__":

+     from runtests import runtests_main

+     runtests_main(["t_setcredopt.py"])

file modified
+102 -4
@@ -11,6 +11,17 @@ 

  import sys

  import time

  

+ testcase_wait = 15

+ cmd_index = 0

+ 

+ debug_all = False

+ debug_gssproxy = False

+ debug_cmd = "gdb --args"

+ debug_cmd_index = -1

+ 

+ valgrind_cmd = "valgrind", "--track-origins=yes"

+ valgrind_everywhere = False

+ 

  try:

      from colorama import Fore, Style

  
@@ -28,6 +39,20 @@ 

      def format_key(status, key):

          return "[" + key + "]"

  

+ def testlib_process_args(args):

+     global debug_all, debug_cmd, debug_cmd_index, debug_gssproxy

+     global testcase_wait, valgrind_cmd, valgrind_everywhere

+ 

+     testcase_wait = args['timeout']

+     debug_cmd_index = args['debug_num']

+ 

+     debug_all = args['debug_all']

+     debug_cmd = args['debug_cmd'] + " "

+     debug_gssproxy = args['debug_gssproxy']

+ 

+     valgrind_cmd = args['valgrind_cmd'] + " "

+     valgrind_everywhere = args['force_valgrind']

+ 

  def print_keyed(status, key, text, io):

      print("%s %s" % (format_key(status, key), text), file=io)

  
@@ -42,7 +67,7 @@ 

  def print_warning(key, text, io=sys.stderr):

      print_keyed("other", key, text, io)

  

- def print_return(ret, name, expected_failure):

+ def print_return(ret, num, name, expected_failure):

      key = "PASS"

      expected = "zero" if not expected_failure else "nonzero"

      if (ret == 0 and expected_failure) or \
@@ -54,9 +79,68 @@ 

      else:

          print_failure(key, "%s test returned %s (expected %s)" %

                        (name, str(ret), expected))

+         if num != -1:

+             print_warning("INFO", "To debug this test case, run:\n" +

+                           ("    make check CHECKARGS='--debug-num=%d'" % num))

  

  WRAP_HOSTNAME = "kdc.gssproxy.dev"

  

+ def run_testcase_cmd(env, conf, cmd, name, expected_failure=False, wait=True):

+     global testcase_wait, debug_cmd_index, cmd_index

+     global valgrind_everywhere, valgrind_cmd, debug_all

+ 

+     logfile = os.path.join(conf['logpath'], "test_%d.log" % cmd_index)

+     logfile = open(logfile, 'w')

+ 

+     print("[NAME]\n%s\n[COMMAND %d]\n%s\n[ENVIRONMENT]\n%s\n\n" % (name,

+           cmd_index, cmd, env), file=logfile)

+     logfile.flush()

+ 

+     testenv = env.copy()

+ 

+     if debug_all or debug_cmd_index == cmd_index:

+         return rundebug_cmd(testenv, conf, cmd, name, expected_failure)

+ 

+     run_cmd = cmd

+     if valgrind_everywhere:

+         run_cmd = valgrind_cmd + cmd

+ 

+     p1 = subprocess.Popen(run_cmd, stderr=subprocess.STDOUT, stdout=logfile,

+                           env=testenv, preexec_fn=os.setsid, shell=True,

+                           executable="/bin/bash")

+ 

+     if not wait:

+         cmd_index += 1

+         conf['prefix'] = str(cmd_index)

+         return p1

+ 

+     try:

+         p1.wait(testcase_wait)

+     except subprocess.TimeoutExpired:

+         # p1.returncode is set to None here

+         if not expected_failure:

+             print_warning("warning", "timeout")

+ 

+     logfile.close()

+     print_return(p1.returncode, cmd_index, "(%d) %s" % (cmd_index, name),

+                  expected_failure)

+     cmd_index += 1

+     conf['prefix'] = str(cmd_index)

+     return p1.returncode if not expected_failure else int(not p1.returncode)

+ 

+ def rundebug_cmd(env, conf, cmd, name, expected_failure=False):

+     global debug_cmd, cmd_index

+ 

+     run_cmd = debug_cmd + cmd

+ 

+     returncode = subprocess.call(run_cmd, env=env, shell=True,

+                                  executable="/bin/bash")

+ 

+     print_return(returncode, cmd_index, "(%d) %s" % (cmd_index, name),

+                  expected_failure)

+     cmd_index += 1

+ 

+     return returncode if not expected_failure else int(not returncode)

  

  def setup_wrappers(base):

  
@@ -633,6 +717,7 @@ 

          f.write(text)

  

  def setup_gssproxy(testdir, logfile, env):

+     global debug_gssproxy, valgrind_cmd

  

      gssproxy = os.path.join(testdir, 'gssproxy')

      if os.path.exists(gssproxy):
@@ -646,9 +731,22 @@ 

  

      socket = os.path.join(gssproxy, 'gp.sock')

      conf = os.path.join(gssproxy, 'gp.conf')

-     gproc = subprocess.Popen(["valgrind", "--track-origins=yes",

-                               "./gssproxy", "-i", "-s", socket, "-c", conf],

+ 

+     cmd = "./gssproxy -i -s " + socket + " -c " + conf

+ 

+     full_command = valgrind_cmd + cmd

+ 

+     if debug_gssproxy:

+         full_command = cmd

+ 

+     gproc = subprocess.Popen(full_command,

                               stdout=logfile, stderr=logfile,

-                              env=gpenv, preexec_fn=os.setsid)

+                              env=gpenv, preexec_fn=os.setsid, shell=True,

+                              executable="/bin/bash")

+ 

+     if debug_gssproxy:

+         print("PID: %d" % (gproc.pid))

+         print("Attach and start debugging, then press enter to continue.")

+         input()

  

      return gproc, socket

PR for enabling debugging of test cases + centralized timeout limits.

Note that t_basic.py requires a pipe between two processes: launching
the test cases as part of run_testcase_cmd is possible, but debugging
either would require creating two named pipes accessible from GDB,
and would prevent re-running of the program without also restarting
the other.

Named pipes could be avoided by using loopback ports, but this doesn't
solve the issue of ease of restarting debugging. I'd probably suggest
that we break t_basic.py out into a stand-alone programs, perhaps
which doesn't rely on sockets/network access, for ease of debugging,
or emulate it with flat-file reading. (The latter might cause issue
with timestamp/ticket validity, but could be worked around by having
a pre-test program create the test case input/output before the test
program is ran...).

NACK. t_basic, and multi-program tests, should be fine. They are easier to read, and allow for debugging using traffic sniffers if needed. I'd honestly prefer one of the following:

  1. Separate commands to latch onto the first or the second (or both)
  2. Launching one in the foreground and one with gdbserver (file is probably better than port for this)
  3. Launching both with gdbserver in the above fashion

Note that we need to be sure that any processes launched in this fashion are killed off when runtests.py finishes.

Keep in mind also that we do already have multiple programs running at all times in the form of the gssproxy daemon and its client(s). (I would also like to see an option for attaching to the gssproxy daemon itself; see above.)

rebased

6 years ago
  • Test failures should cause a message to be printed about how to debug that test case.
  • In the future (may or may not be in scope for this PR) we may want options for just running that one test case.
  • There should be a way to pass in flags to make check (e.g., through variables, like make check DEBUG=16 or so).
  • I'd like to see an option for running the full valgrind leak checker
  • Test suite should indicate at the start of the run that it has flags at all, and how to get the full set of options
  • I don't understand the purpose of --noisy-timeout.

(Please comment when you force-push; pagure doesn't generate an event for that.)

Thanks for the feedback :)

  • I'll implement that.
  • I'll see how that goes; if its easy, I'll bundle it into this PR.
  • make check flags, iirc, aren't supported by krb5. At this time, after running an appropriate make, we can directly call python3 tests/runtests.py with the appropriate options we wanted. However, I'll take a look at that.
  • I'll add an option for controlling valgrind flags, and an option for forcing valgrind on all test cases (not just gssproxy)
  • Okay
  • Noisy Timeout is more of an enhancement in the style of the other pull request technically; its meant to make timing out explicitly clear to new runners of the testsuite. Printing returned value == None to the runners doesn't explicitly say that the test timed out without them reading the comments in the code.
  • make check CHECKARGS="--debug-num=4 --force-valgrind" is the how to run the test suite with args now

  • Sets of tests can be run individually now. So, e.g., python3 tests/t_acquire.py works, along with all of the options under python3 tests/runtests.py. Note that numbering for test cases is per-running-unit, so the test case number under python3 tests/runtests.py differs from python3 tests/t_basic.py for instance.

rebased

6 years ago
  • I'd like the message printed at the start to use CHECKARGS. make check is useful here because it will update dependencies and I don't want to deal with debugging when they're not. There's therefore no need for the instructions on how to run it "directly".
  • Similarly, use make check CHECKARGS= in the debug messages. (If this really offends you, I'm also okay with just removing the "to runtests.py" part.)
  • Please remove noisy timeout. If you want, you can print something more useful when tests time out and failure was not expected; otherwise, timeout isn't interesting.
  • While looking for valgrind output from --force-valgrind, I noticed that the test numbers as displayed do not match the numbers of the log files. Please reconcile the two.

rebased

6 years ago
  • make check should now be listed everywhere
  • make check CHECKARGS='--debug-num=<x>' now listed on error
  • --noisy-timeout removed, but info message printed still
  • All logs are per-test case now. Again, numbering is not stable when running only portions of the test suite. Also, short of lovely hackery to preserve the t_<name>.py name, I've decided to go with the simpler $testdir/test_<x>.log naming scheme.
  • Shorten the message on initial start. Two of the lines are currently redundant, and an earlier requested removal has not been carried out.
  • Per our discussion on IRC and comments above, timeout is not interesting unless it was unexpected. (Test 18 generates an expected timeout.)
  • In t_basic.py, there are comments added explaining what p1 and p2 are. Remove them please.

Once these are addressed, I have no further feedback, but we should probably get input from @simo here too.

rebased

6 years ago

Okay, in the course of using it, I realized that python's/argparse's nargs="+" option is incompatible with passing argument-like things. That is, the following all fail:

CHECKARGS='--valgrind-cmd="valgrind --track-origins=yes -v --leak-check=full --show-leak-kinds=all"'
CHECKARGS='--valgrind-cmd="valgrind,--track-origins=yes,-v,--leak-check=full,--show-leak-kinds=all"'
CHECKARGS='--valgrind-cmd=valgrind --valgrind-cmd=--track-origins=yes --valgrind-cmd=-v --valgrind-cmd=--leak-check=full --valgrind-cmd=--show-leak-kinds=all'

(The latter being most desirable if a touch verbose; but it looks to be discussed here)

I propose that we instead make --valgrind-cmd and --debug-cmd a string, and split on either spaces or commas. Thoughts?

rebased

6 years ago

(Per IRC discussion: use a shell when calling subprocess to avoid parsing strings in argparse)

rebased

6 years ago

rebased

6 years ago

(Per IRC discussion: please pass the TERM variable through to gdb so that TUI mode works, and also fix whatever causes breakage on Debian.)

rebased

6 years ago

rebased

6 years ago

rebased

6 years ago

TERM is passed through, and afaict, executable="/bin/bash" fixes the issue on Debian. Also, all of the artifacts in $testdir now are numbered as expected.

Let's actually do this right.

Thanks for fixing those issues. However, the test artifact numbering is still slightly inconsistent - sometimes there's a leading '0', sometimes not. (I'd recommend removing the leading 0, unless you also want to fix the printed numbers.) One additional comment inline.

rebased

6 years ago

rebased

6 years ago

Commit 7917a46 fixes this pull-request

Pull-Request has been merged by ascheel@redhat.com

6 years ago

Commit 5a0d500 fixes this pull-request

Pull-Request has been merged by ascheel@redhat.com

6 years ago

Commit 7917a46 fixes this pull-request

Pull-Request has been merged by ascheel@redhat.com

6 years ago