djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

run_tests: Parallelism improvements, partial sync with micropython

Browse Source 
Take changes from https://github.com/micropython/micropython/pull/3694
(expected to be merged soon) as well as other accumulated stuff from
upstream that we want.

Leave our desired differences, including:
 * silencing warnings in python3
 * renaming the file descriptors returned by openpty()
 * adding ulab tests
 * Adding "." to the import path for skip_if

This speeds up `make test_full` and should also reduce the time in CI
a little bit.
This commit is contained in:
Jeff Epler 2021-05-13 10:33:24 -05:00
parent 47947dc23c
commit e6fe55349d
2 changed files with 43 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ports/unix/Makefile
View File

@ -228,7 +228,7 @@ include $(TOP)/py/mkrules.mk
test: $(PROG) $(TOP)/tests/run-tests.py test: $(PROG) $(TOP)/tests/run-tests.py
$(eval DIRNAME=ports/$(notdir $(CURDIR))) $(eval DIRNAME=ports/$(notdir $(CURDIR)))
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --auto-jobs cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py
test_full: $(PROG) $(TOP)/tests/run-tests.py test_full: $(PROG) $(TOP)/tests/run-tests.py
$(eval DIRNAME=ports/$(notdir $(CURDIR))) $(eval DIRNAME=ports/$(notdir $(CURDIR)))

96
tests/run-tests.py
View File

@ -7,10 +7,11 @@ import platform
import argparse import argparse
import inspect import inspect
import re import re
import threading from glob import glob
import multiprocessing import multiprocessing
from multiprocessing.pool import ThreadPool from multiprocessing.pool import ThreadPool
from glob import glob import threading
import tempfile
# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0] # See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0]
# are guaranteed to always work, this one should though. # are guaranteed to always work, this one should though.
@ -169,26 +170,27 @@ def run_micropython(pyb, args, test_file, is_special=False):
# if running via .mpy, first compile the .py file # if running via .mpy, first compile the .py file
if args.via_mpy: if args.via_mpy:
mpy_modname = tempfile.mktemp(dir="")
mpy_filename = mpy_modname + ".mpy"
subprocess.check_output( subprocess.check_output(
[MPYCROSS] [MPYCROSS]
+ args.mpy_cross_flags.split() + args.mpy_cross_flags.split()
+ ["-o", "mpytest.mpy", "-X", "emit=" + args.emit, test_file] + ["-o", mpy_filename, "-X", "emit=" + args.emit, test_file]
) )
cmdlist.extend(["-m", "mpytest"]) cmdlist.extend(["-m", mpy_modname])
else: else:
cmdlist.append(test_file) cmdlist.append(test_file)
# run the actual test # run the actual test
e = {"LANG": "en_US.UTF-8", "MICROPYPATH": os.environ["MICROPYPATH"]}
try: try:
output_mupy = subprocess.check_output(cmdlist, env=e, stderr=subprocess.STDOUT) output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as error: except subprocess.CalledProcessError as er:
had_crash = True had_crash = True
output_mupy = error.output + b"CRASH" output_mupy = er.output + b"CRASH"
# clean up if we had an intermediate .mpy file # clean up if we had an intermediate .mpy file
if args.via_mpy: if args.via_mpy:
rm_f("mpytest.mpy") rm_f(mpy_filename)
else: else:
# run on pyboard # run on pyboard
@ -200,7 +202,7 @@ def run_micropython(pyb, args, test_file, is_special=False):
if not is_special and e.args[0] == "exception": if not is_special and e.args[0] == "exception":
output_mupy = e.args[1] + e.args[2] + b"CRASH" output_mupy = e.args[1] + e.args[2] + b"CRASH"
else: else:
output_mupy = b"CRASH" output_mupy = bytes(e.args[0], "ascii") + b"\nCRASH"
# canonical form for all ports/platforms is to use \n for end-of-line # canonical form for all ports/platforms is to use \n for end-of-line
output_mupy = output_mupy.replace(b"\r\n", b"\n") output_mupy = output_mupy.replace(b"\r\n", b"\n")
@ -241,10 +243,9 @@ def run_micropython(pyb, args, test_file, is_special=False):
else: else:
# a regex # a regex
if lines_exp[i][1].match(lines_mupy[i_mupy]): if lines_exp[i][1].match(lines_mupy[i_mupy]):
# print("match", lines_exp[i][0], lines_mupy[i_mupy])
lines_mupy[i_mupy] = lines_exp[i][0] lines_mupy[i_mupy] = lines_exp[i][0]
else: else:
# print("don't match: %r %s" % (lines_exp[i][0], lines_mupy[i_mupy])) # DEBUG # print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
pass pass
i_mupy += 1 i_mupy += 1
if i_mupy >= len(lines_mupy): if i_mupy >= len(lines_mupy):
@ -266,6 +267,9 @@ class ThreadSafeCounter:
self._value = start self._value = start
self._lock = threading.Lock() self._lock = threading.Lock()
def increment(self):
self.add(1)
def add(self, to_add): def add(self, to_add):
with self._lock: with self._lock:
self._value += to_add self._value += to_add
@ -306,11 +310,11 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
if not (args.list_tests or args.write_exp): if not (args.list_tests or args.write_exp):
# Even if we run completely different tests in a different directory, # Even if we run completely different tests in a different directory,
# we need to access feature_checks from the same directory as the # we need to access feature_checks from the same directory as the
# run-tests script itself so use base_path. # run-tests.py script itself so use base_path.
# Check if micropython.native is supported, and skip such tests if it's not # Check if micropython.native is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, "native_check.py") output = run_feature_check(pyb, args, base_path, "native_check.py")
if output.endswith(b"CRASH"): if output != b"native\n":
skip_native = True skip_native = True
# Check if arbitrary-precision integers are supported, and skip such tests if it's not # Check if arbitrary-precision integers are supported, and skip such tests if it's not
@ -325,7 +329,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
# Check if set type (and set literals) is supported, and skip such tests if it's not # Check if set type (and set literals) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, "set_check.py") output = run_feature_check(pyb, args, base_path, "set_check.py")
if output.endswith(b"CRASH"): if output != b"{1}\n":
skip_set_type = True skip_set_type = True
# Check if slice is supported, and skip such tests if it's not # Check if slice is supported, and skip such tests if it's not
@ -335,12 +339,12 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
# Check if async/await keywords are supported, and skip such tests if it's not # Check if async/await keywords are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, "async_check.py") output = run_feature_check(pyb, args, base_path, "async_check.py")
if output.endswith(b"CRASH"): if output != b"async\n":
skip_async = True skip_async = True
# Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not # Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, "const.py") output = run_feature_check(pyb, args, base_path, "const.py")
if output.endswith(b"CRASH"): if output != b"1\n":
skip_const = True skip_const = True
# Check if __rOP__ special methods are supported, and skip such tests if it's not # Check if __rOP__ special methods are supported, and skip such tests if it's not
@ -365,10 +369,10 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
upy_byteorder = run_feature_check(pyb, args, base_path, "byteorder.py") upy_byteorder = run_feature_check(pyb, args, base_path, "byteorder.py")
upy_float_precision = run_feature_check(pyb, args, base_path, "float.py") upy_float_precision = run_feature_check(pyb, args, base_path, "float.py")
if upy_float_precision.endswith(b"CRASH"): try:
upy_float_precision = 0
else:
upy_float_precision = int(upy_float_precision) upy_float_precision = int(upy_float_precision)
except ValueError:
upy_float_precision = 0
has_complex = run_feature_check(pyb, args, base_path, "complex.py") == b"complex\n" has_complex = run_feature_check(pyb, args, base_path, "complex.py") == b"complex\n"
has_coverage = run_feature_check(pyb, args, base_path, "coverage.py") == b"coverage\n" has_coverage = run_feature_check(pyb, args, base_path, "coverage.py") == b"coverage\n"
cpy_byteorder = subprocess.check_output( cpy_byteorder = subprocess.check_output(
@ -545,7 +549,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray") is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray")
is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset") is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")
is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests
is_async = test_name.startswith("async_") is_async = test_name.startswith(("async_", "uasyncio_"))
is_const = test_name.startswith("const") is_const = test_name.startswith("const")
is_io_module = test_name.startswith("io_") is_io_module = test_name.startswith("io_")
@ -610,8 +614,8 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
filename_mupy = os.path.join(result_dir, test_basename + ".out") filename_mupy = os.path.join(result_dir, test_basename + ".out")
if output_expected == output_mupy: if output_expected == output_mupy:
# print("pass ", test_file) print("pass ", test_file)
passed_count.add(1) passed_count.increment()
rm_f(filename_expected) rm_f(filename_expected)
rm_f(filename_mupy) rm_f(filename_mupy)
else: else:
@ -619,17 +623,13 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
f.write(output_expected) f.write(output_expected)
with open(filename_mupy, "wb") as f: with open(filename_mupy, "wb") as f:
f.write(output_mupy) f.write(output_mupy)
print("### Expected")
print(output_expected)
print("### Actual")
print(output_mupy)
print("FAIL ", test_file) print("FAIL ", test_file)
failed_tests.append(test_file) failed_tests.append(test_name)
test_count.add(1) test_count.increment()
if args.list_tests: if pyb or args.list_tests:
return True num_threads = 1
if num_threads > 1: if num_threads > 1:
pool = ThreadPool(num_threads) pool = ThreadPool(num_threads)
@ -638,6 +638,9 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
for test in tests: for test in tests:
run_one_test(test) run_one_test(test)
if args.list_tests:
return True
print( print(
"{} tests performed ({} individual testcases)".format( "{} tests performed ({} individual testcases)".format(
test_count.value, testcase_count.value test_count.value, testcase_count.value
@ -645,18 +648,12 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
) )
print("{} tests passed".format(passed_count.value)) print("{} tests passed".format(passed_count.value))
if len(skipped_tests.value) > 0: skipped_tests = sorted(skipped_tests.value)
print( if len(skipped_tests) > 0:
"{} tests skipped: {}".format( print("{} tests skipped: {}".format(len(skipped_tests), " ".join(skipped_tests)))
len(skipped_tests.value), " ".join(sorted(skipped_tests.value)) failed_tests = sorted(failed_tests.value)
) if len(failed_tests) > 0:
) print("{} tests failed: {}".format(len(failed_tests), " ".join(failed_tests)))
if len(failed_tests.value) > 0:
print(
"{} tests failed: {}".format(
len(failed_tests.value), " ".join(sorted(failed_tests.value))
)
)
return False return False
# all tests succeeded # all tests succeeded
@ -758,18 +755,11 @@ the last matching regex is used:
cmd_parser.add_argument( cmd_parser.add_argument(
"-j", "-j",
"--jobs", "--jobs",
default=1, default=multiprocessing.cpu_count(),
metavar="N", metavar="N",
type=int, type=int,
help="Number of tests to run simultaneously", help="Number of tests to run simultaneously",
) )
cmd_parser.add_argument(
"--auto-jobs",
action="store_const",
dest="jobs",
const=multiprocessing.cpu_count(),
help="Set the -j values to the CPU (thread) count",
)
cmd_parser.add_argument("files", nargs="*", help="input test files") cmd_parser.add_argument("files", nargs="*", help="input test files")
cmd_parser.add_argument( cmd_parser.add_argument(
"--print-failures", "--print-failures",
@ -869,9 +859,7 @@ the last matching regex is used:
if not args.keep_path: if not args.keep_path:
# clear search path to make sure tests use only builtin modules and those in extmod # clear search path to make sure tests use only builtin modules and those in extmod
os.environ["MICROPYPATH"] = ( os.environ["MICROPYPATH"] = os.pathsep + base_path("../extmod")
os.pathsep + base_path("../extmod") + os.pathsep + base_path(".")
)
try: try:
os.makedirs(args.result_dir, exist_ok=True) os.makedirs(args.result_dir, exist_ok=True)