circuitpython/tests/run-tests

242 lines
9.9 KiB
Plaintext
Raw Normal View History

#! /usr/bin/env python3
2014-01-27 16:53:28 -05:00
import os
import subprocess
import sys
import platform
import argparse
2014-01-27 16:53:28 -05:00
from glob import glob
# Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
2014-01-27 16:53:28 -05:00
if os.name == 'nt':
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../windows/micropython.exe')
2014-01-27 16:53:28 -05:00
else:
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../unix/micropython')
2014-01-27 16:53:28 -05:00
# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
2014-12-11 17:51:39 -05:00
os.environ['PYTHONIOENCODING'] = 'utf-8'
os.environ['MICROPYPATH'] = ''
def rm_f(fname):
if os.path.exists(fname):
os.remove(fname)
def run_micropython(pyb, args, test_file):
if pyb is None:
# run on PC
if test_file.startswith('cmdline/'):
# special handling for tests of the unix cmdline program
# check for any cmdline options needed for this test
args = [MICROPYTHON]
with open(test_file, 'rb') as f:
line = f.readline()
if line.startswith(b'# cmdline:'):
args += line[10:].strip().split()
# run the test, possibly with redirected input
try:
if test_file.startswith('cmdline/repl_'):
f = open(test_file, 'rb')
output_mupy = subprocess.check_output(args, stdin=f)
f.close()
else:
output_mupy = subprocess.check_output(args + [test_file])
except subprocess.CalledProcessError:
output_mupy = b'CRASH'
# erase parts of the output that are not stable across runs
with open(test_file + '.exp', 'rb') as f:
lines_exp = f.readlines()
lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]
if output_mupy.endswith(b'\n'):
lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
if len(lines_mupy) == len(lines_exp):
for i in range(len(lines_mupy)):
pos = lines_exp[i].find(b'######')
if pos != -1 and len(lines_mupy[i]) >= pos:
lines_mupy[i] = lines_mupy[i][:pos] + b'######\n'
output_mupy = b''.join(lines_mupy)
else:
# a standard test
try:
output_mupy = subprocess.check_output([MICROPYTHON, '-X', 'emit=' + args.emit, test_file])
except subprocess.CalledProcessError:
output_mupy = b'CRASH'
else:
# run on pyboard
import pyboard
pyb.enter_raw_repl()
try:
output_mupy = pyb.execfile(test_file).replace(b'\r\n', b'\n')
except pyboard.PyboardError:
output_mupy = b'CRASH'
return output_mupy
def run_tests(pyb, tests, args):
test_count = 0
testcase_count = 0
passed_count = 0
failed_tests = []
skipped_tests = []
skip_tests = set()
skip_native = False
# Check if micropython.native is supported, and skip such tests if it's not
native = run_micropython(pyb, args, 'micropython/native_check.py')
if native == b'CRASH':
skip_native = True
# Some tests shouldn't be run under Travis CI
if os.getenv('TRAVIS') == 'true':
skip_tests.add('basics/memoryerror.py')
# Some tests shouldn't be run on pyboard
if pyb is not None:
skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead
skip_tests.add('float/float2int_doubleprec.py') # requires double precision floating point to work
# Some tests are known to fail on 64-bit machines
if pyb is None and platform.architecture()[0] == '64bit':
pass
# Some tests use unsupported features on Windows
if os.name == 'nt':
skip_tests.add('import\\import_file.py') #works but CPython prints forward slashes
# Some tests are known to fail with native emitter
# Remove them from the below when they work
if args.emit == 'native':
skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class class_super class_super_object closure1 closure2 closure_defargs del_deref del_local fun3 fun_calldblstar fun_callstar fun_callstardblstar fun_defargs fun_defargs2 fun_kwargs fun_kwonly fun_kwonlydef fun_kwvarargs fun_varargs gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_throw generator1 generator2 generator_args generator_close generator_closure generator_exc generator_return generator_send globals_del string_format string_join subclass_native2_list subclass_native2_tuple try_finally_loops try_finally_return try_reraise try_reraise2 unboundlocal with1 with_break with_continue with_return'.split()})
skip_tests.add('basics/boundmeth1.py') # requires support for many args
skip_tests.add('float/string_format.py')
skip_tests.add('float/cmath_fun.py') # requires f(*args) support
skip_tests.add('import/gen_context.py')
skip_tests.add('io/file_with.py')
skip_tests.add('micropython/heapalloc.py')
skip_tests.add('misc/features.py')
skip_tests.add('misc/recursion.py')
skip_tests.add('misc/rge_sm.py')
skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info
for test_file in tests:
test_basename = os.path.basename(test_file)
test_name = os.path.splitext(test_basename)[0]
is_native = test_name.startswith("native_") or test_name.startswith("viper_")
if test_file in skip_tests or (skip_native and is_native):
print("skip ", test_file)
skipped_tests.append(test_name)
continue
# get expected output
test_file_expected = test_file + '.exp'
if os.path.isfile(test_file_expected):
# expected output given by a file, so read that in
with open(test_file_expected, 'rb') as f:
output_expected = f.read()
if os.name == 'nt':
output_expected = output_expected.replace(b'\n', b'\r\n')
else:
# run CPython to work out expected output
try:
output_expected = subprocess.check_output([CPYTHON3, '-B', test_file])
if args.write_exp:
with open(test_file_expected, 'wb') as f:
f.write(output_expected)
except subprocess.CalledProcessError:
output_expected = b'CPYTHON3 CRASH'
if args.write_exp:
continue
# run Micro Python
output_mupy = run_micropython(pyb, args, test_file)
if os.name != 'nt':
# It may be the case that we run Windows build under Linux
# (using Wine).
output_mupy = output_mupy.replace(b'\r\n', b'\n')
if output_mupy == b'SKIP\n' or output_mupy == b'SKIP\r\n':
print("skip ", test_file)
skipped_tests.append(test_name)
continue
testcase_count += len(output_expected.splitlines())
filename_expected = test_basename + ".exp"
filename_mupy = test_basename + ".out"
if output_expected == output_mupy:
print("pass ", test_file)
passed_count += 1
rm_f(filename_expected)
rm_f(filename_mupy)
else:
with open(filename_expected, "wb") as f:
f.write(output_expected)
with open(filename_mupy, "wb") as f:
f.write(output_mupy)
print("FAIL ", test_file)
failed_tests.append(test_name)
test_count += 1
print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
print("{} tests passed".format(passed_count))
if len(skipped_tests) > 0:
print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests)))
if len(failed_tests) > 0:
print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests)))
return False
# all tests succeeded
return True
def main():
cmd_parser = argparse.ArgumentParser(description='Run tests for Micro Python.')
cmd_parser.add_argument('--pyboard', action='store_true', help='run the tests on the pyboard')
cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device of the pyboard')
cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython')
cmd_parser.add_argument('--emit', default='bytecode', help='Micro Python emitter to use (bytecode or native)')
cmd_parser.add_argument('files', nargs='*', help='input test files')
args = cmd_parser.parse_args()
if args.pyboard:
import pyboard
pyb = pyboard.Pyboard(args.device)
pyb.enter_raw_repl()
else:
pyb = None
if len(args.files) == 0:
if args.test_dirs is None:
if pyb is None:
# run PC tests
test_dirs = ('basics', 'micropython', 'float', 'import', 'io', 'misc', 'unicode', 'extmod', 'unix', 'cmdline')
else:
# run pyboard tests
test_dirs = ('basics', 'micropython', 'float', 'misc', 'extmod', 'pyb', 'pybnative', 'inlineasm')
else:
# run tests from these directories
test_dirs = args.test_dirs
tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
2014-01-27 16:53:28 -05:00
else:
# tests explicitly given
tests = args.files
2014-01-27 16:53:28 -05:00
if not run_tests(pyb, tests, args):
sys.exit(1)
2014-01-27 16:53:28 -05:00
if __name__ == "__main__":
main()