From 344ff83156c03585944d55465b5cc44d447d3ff2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 19 2018 10:07:49 +0000 Subject: [PATCH 1/3] Reduce code duplication + a couple of small fixes Signed-off-by: Pierre-Yves Chibon --- diff --git a/runpaguretests.py b/runpaguretests.py index 4d95b38..1f486e0 100755 --- a/runpaguretests.py +++ b/runpaguretests.py @@ -226,64 +226,7 @@ def do_run(args): continue suites.append(fname.replace(".py", "")) - global PRINTLOCK - PRINTLOCK = threading.RLock() - global NUMREMAINING - NUMREMAINING = len(suites) - - sem = threading.BoundedSemaphore(NUMPROCS) - - # Create a worker per test - workers = {} - for suite in suites: - workers[suite] = WorkerThread(sem, suite, results=args.results) - - # Start the workers - print("Starting the workers") - print() - print() - for worker in workers.values(): - worker.start() - - # Wait for them to terminate - for worker in workers: - workers[worker].join() - print_running() - print() - print("All work done") - - # Gather results - print() - print() - print("RESULTS:") - failed = [] - for worker in workers: - if not workers[worker].failed: - result = "Success" - else: - result = "FAILED" - failed.append(worker) - print("Test %s result: %s" % (worker, result)) - - # Write failed - if failed: - with open(os.path.join(args.results, "newfailed"), "w") as ffile: - ffile.write(json.dumps(failed)) - - # Stats - end = time.time() - print() - print() - print( - "Ran %d tests in %f seconds, of which %d failed" - % (len(workers), (end - start), len(failed)) - ) - - # Exit - if len(failed) == 0: - print("ALL PASSED! CONGRATULATIONS!") - else: - return 1 + _run_test_suites(suites) def do_rerun(args): @@ -323,6 +266,10 @@ def do_rerun(args): print("File containing the failed tests is not JSON") return 1 + _run_test_suites(suites) + + +def _run_test_suites(suites): global PRINTLOCK PRINTLOCK = threading.RLock() global NUMREMAINING @@ -401,13 +348,13 @@ def main(): return_code = 0 try: - arg.func(arg) + return_code = arg.func(arg) except KeyboardInterrupt: print("\nInterrupted by user.") return_code = 1 except Exception as err: print('Error: {0}'.format(err)) - logging.exception("Generic error catched:") + logging.exception("Generic error caught:") return_code = 5 return return_code From ad115eaf4a03bb6147c1ae3f82939603f28e76e5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 19 2018 10:13:03 +0000 Subject: [PATCH 2/3] Import the py2/py3 parallel runner Signed-off-by: Pierre-Yves Chibon --- diff --git a/runpaguretests.py b/runpaguretests.py index 1f486e0..15c7144 100755 --- a/runpaguretests.py +++ b/runpaguretests.py @@ -45,7 +45,14 @@ def setup_parser(): dest="py2", action="store_true", default=False, - help="Runs the tests in python2 instead of python3", + help="Runs the tests only in python2 instead of both python2 and python3", + ) + parser_run.add_argument( + "--py3", + dest="py3", + action="store_true", + default=False, + help="Runs the tests only in python3 instead of both python2 and python3", ) parser_run.add_argument( "--results", @@ -137,7 +144,7 @@ def remove_running(suite, failed): class WorkerThread(threading.Thread): - def __init__(self, sem, pyver, suite): + def __init__(self, sem, pyver, suite, results): name = "py%d-%s" % (pyver, suite) super(WorkerThread, self).__init__(name="worker-%s" % name) self.name = name @@ -150,7 +157,7 @@ class WorkerThread(threading.Thread): def run(self): with self.sem: add_running(self.name) - with open(os.path.join(args.results, self.name), "w") as resfile: + with open(os.path.join(self.results, self.name), "w") as resfile: if self.pyver == 2: runner = RUNNER_PY2 elif self.pyver == 3: @@ -193,12 +200,6 @@ def do_run(args): print("Using %d processes" % NUMPROCS) - print("Start timing") - start = time.time() - - if args.py2: - RUNNER = RUNNER_PY2 - suites = [] if args.failed_tests: @@ -226,7 +227,7 @@ def do_run(args): continue suites.append(fname.replace(".py", "")) - _run_test_suites(suites) + _run_test_suites(args, suites) def do_rerun(args): @@ -250,12 +251,6 @@ def do_rerun(args): print("Using %d processes" % NUMPROCS) - print("Start timing") - start = time.time() - - if args.py2: - RUNNER = RUNNER_PY2 - suites = [] print("Loading failed tests") @@ -266,10 +261,13 @@ def do_rerun(args): print("File containing the failed tests is not JSON") return 1 - _run_test_suites(suites) + _run_test_suites(args, suites) -def _run_test_suites(suites): +def _run_test_suites(args, suites): + print("Start timing") + start = time.time() + global PRINTLOCK PRINTLOCK = threading.RLock() global NUMREMAINING @@ -279,8 +277,17 @@ def _run_test_suites(suites): # Create a worker per test workers = {} + pyvers = (2, 3) + if args.py2: + pyvers = (2,) + elif args.py3: + pyvers = (3,) + for suite in suites: - workers[suite] = WorkerThread(sem, suite, results=args.results) + for pyver in pyvers: + NUMREMAINING += 1 + workers["py%d-%s" % (pyver, suite)] = WorkerThread( + sem, pyver, suite, results=args.results) # Start the workers print("Starting the workers") From 3754e1b808762ab8fc539a3e23a25eff4ea15426 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sep 19 2018 10:14:52 +0000 Subject: [PATCH 3/3] Don't over estimate the number remaining Signed-off-by: Pierre-Yves Chibon --- diff --git a/runpaguretests.py b/runpaguretests.py index 15c7144..3b49314 100755 --- a/runpaguretests.py +++ b/runpaguretests.py @@ -271,7 +271,7 @@ def _run_test_suites(args, suites): global PRINTLOCK PRINTLOCK = threading.RLock() global NUMREMAINING - NUMREMAINING = len(suites) + NUMREMAINING = 0 sem = threading.BoundedSemaphore(NUMPROCS)