2014-04-02 10:31:39 -04:00
#! /usr/bin/env python3
2014-01-27 16:53:28 -05:00
import os
import subprocess
import sys
2014-09-17 17:56:34 -04:00
import platform
2014-05-03 11:43:27 -04:00
import argparse
2020-04-08 02:01:58 -04:00
import inspect
2015-03-20 13:25:25 -04:00
import re
2014-01-27 16:53:28 -05:00
from glob import glob
2020-04-08 02:01:58 -04:00
# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0]
# are guaranteed to always work, this one should though.
BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None)))
def base_path(*p):
return os.path.abspath(os.path.join(BASEPATH, *p)).replace('\\', '/')
2014-04-04 10:28:34 -04:00
# Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
2014-01-27 16:53:28 -05:00
if os.name == 'nt':
2014-04-04 10:28:34 -04:00
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
2020-04-08 02:01:58 -04:00
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/windows/micropython.exe'))
2014-01-27 16:53:28 -05:00
else:
2014-04-03 15:06:35 -04:00
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
2020-04-08 02:01:58 -04:00
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/unix/micropython'))
2014-01-27 16:53:28 -05:00
2016-09-19 22:19:35 -04:00
# mpy-cross is only needed if --via-mpy command-line arg is passed
2020-04-08 02:01:58 -04:00
MPYCROSS = os.getenv('MICROPY_MPYCROSS', base_path('../mpy-cross/mpy-cross'))
2016-09-19 22:19:35 -04:00
2020-03-30 04:14:09 -04:00
# For diff'ing test output
DIFF = os.getenv('MICROPY_DIFF', 'diff -u')
2014-11-19 10:44:31 -05:00
# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
2014-12-11 17:51:39 -05:00
os.environ['PYTHONIOENCODING'] = 'utf-8'
2014-11-19 10:44:31 -05:00
2014-03-22 19:07:30 -04:00
def rm_f(fname):
if os.path.exists(fname):
os.remove(fname)
2016-07-26 14:29:48 -04:00
# unescape wanted regex chars and escape unwanted ones
def convert_regex_escapes(line):
cs = []
escape = False
for c in str(line, 'utf8'):
if escape:
escape = False
cs.append(c)
elif c == '\\':
escape = True
elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'):
cs.append('\\' + c)
else:
cs.append(c)
# accept carriage-return(s) before final newline
if cs[-1] == '\n':
cs[-1] = '\r*\n'
return bytes(''.join(cs), 'utf8')
2017-04-14 10:07:13 -04:00
def run_micropython(pyb, args, test_file, is_special=False):
2017-12-04 18:52:41 -05:00
special_tests = (
'micropython/meminfo.py', 'basics/bytes_compare3.py',
'basics/builtin_help.py', 'thread/thread_exc2.py',
2020-04-02 01:59:08 -04:00
'esp32/partition_ota.py',
2017-12-04 18:52:41 -05:00
)
2018-02-14 01:19:44 -05:00
had_crash = False
2014-10-05 17:27:12 -04:00
if pyb is None:
# run on PC
2020-04-08 02:01:58 -04:00
if test_file.startswith(('cmdline/', base_path('feature_check/'))) or test_file in special_tests:
2015-03-13 06:58:34 -04:00
# special handling for tests of the unix cmdline program
2016-07-26 14:29:48 -04:00
is_special = True
2015-03-13 06:58:34 -04:00
2017-04-14 10:07:13 -04:00
if is_special:
2015-03-13 06:58:34 -04:00
# check for any cmdline options needed for this test
args = [MICROPYTHON]
with open(test_file, 'rb') as f:
line = f.readline()
if line.startswith(b'# cmdline:'):
2015-08-06 06:34:48 -04:00
# subprocess.check_output on Windows only accepts strings, not bytes
args += [str(c, 'utf-8') for c in line[10:].strip().split()]
2015-03-13 06:58:34 -04:00
# run the test, possibly with redirected input
try:
2015-08-30 04:36:42 -04:00
if 'repl_' in test_file:
2015-07-23 15:05:30 -04:00
# Need to use a PTY to test command line editing
2015-08-06 06:34:48 -04:00
try:
import pty
except ImportError:
# in case pty module is not available, like on Windows
return b'SKIP\n'
2015-07-23 15:05:30 -04:00
import select
2015-11-24 12:52:03 -05:00
def get(required=False):
2015-07-23 15:05:30 -04:00
rv = b''
while True:
ready = select.select([master], [], [], 0.02)
if ready[0] == [master]:
rv += os.read(master, 1024)
else:
2015-11-24 12:52:03 -05:00
if not required or rv:
return rv
2015-07-23 15:05:30 -04:00
def send_get(what):
os.write(master, what)
return get()
with open(test_file, 'rb') as f:
# instead of: output_mupy = subprocess.check_output(args, stdin=f)
master, slave = pty.openpty()
p = subprocess.Popen(args, stdin=slave, stdout=slave,
stderr=subprocess.STDOUT, bufsize=0)
2015-11-24 12:52:03 -05:00
banner = get(True)
2015-07-23 15:05:30 -04:00
output_mupy = banner + b''.join(send_get(line) for line in f)
2016-12-19 23:13:49 -05:00
send_get(b'\x04') # exit the REPL, so coverage info is saved
2019-03-31 03:47:11 -04:00
# At this point the process might have exited already, but trying to
# kill it 'again' normally doesn't result in exceptions as Python and/or
# the OS seem to try to handle this nicely. When running Linux on WSL
# though, the situation differs and calling Popen.kill after the process
# terminated results in a ProcessLookupError. Just catch that one here
# since we just want the process to be gone and that's the case.
try:
p.kill()
except ProcessLookupError:
pass
2015-07-23 15:05:30 -04:00
os.close(master)
os.close(slave)
2015-03-13 06:58:34 -04:00
else:
2017-09-24 21:15:48 -04:00
output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT)
2015-03-13 06:58:34 -04:00
except subprocess.CalledProcessError:
2015-09-21 06:24:17 -04:00
return b'CRASH'
2015-03-13 06:58:34 -04:00
else:
2016-09-19 22:19:35 -04:00
# a standard test run on PC
# create system command
cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit]
if args.heapsize is not None:
cmdlist.extend(['-X', 'heapsize=' + args.heapsize])
# if running via .mpy, first compile the .py file
if args.via_mpy:
2019-12-22 07:51:57 -05:00
subprocess.check_output([MPYCROSS] + args.mpy_cross_flags.split() + ['-o', 'mpytest.mpy', '-X', 'emit=' + args.emit, test_file])
2016-09-19 22:19:35 -04:00
cmdlist.extend(['-m', 'mpytest'])
else:
2016-03-15 09:04:43 -04:00
cmdlist.append(test_file)
2016-09-19 22:19:35 -04:00
# run the actual test
try:
2017-09-24 21:15:48 -04:00
output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT)
2018-02-14 01:19:44 -05:00
except subprocess.CalledProcessError as er:
had_crash = True
output_mupy = er.output + b'CRASH'
2016-09-19 22:19:35 -04:00
# clean up if we had an intermediate .mpy file
if args.via_mpy:
rm_f('mpytest.mpy')
2014-10-05 17:27:12 -04:00
else:
# run on pyboard
pyb.enter_raw_repl()
try:
2016-03-07 07:01:21 -05:00
output_mupy = pyb.execfile(test_file)
2018-07-18 09:45:23 -04:00
except pyboard.PyboardError as e:
2018-02-14 01:19:44 -05:00
had_crash = True
2018-07-18 09:45:23 -04:00
if not is_special and e.args[0] == 'exception':
output_mupy = e.args[1] + e.args[2] + b'CRASH'
else:
output_mupy = b'CRASH'
2014-10-05 17:27:12 -04:00
2016-03-07 07:01:21 -05:00
# canonical form for all ports/platforms is to use \n for end-of-line
output_mupy = output_mupy.replace(b'\r\n', b'\n')
2017-01-23 04:44:29 -05:00
# don't try to convert the output if we should skip this test
2018-02-14 01:19:44 -05:00
if had_crash or output_mupy in (b'SKIP\n', b'CRASH'):
2017-01-23 04:44:29 -05:00
return output_mupy
2016-07-26 14:29:48 -04:00
if is_special or test_file in special_tests:
# convert parts of the output that are not stable across runs
with open(test_file + '.exp', 'rb') as f:
lines_exp = []
for line in f.readlines():
if line == b'########\n':
line = (line,)
else:
line = (line, re.compile(convert_regex_escapes(line)))
lines_exp.append(line)
lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]
if output_mupy.endswith(b'\n'):
lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
i_mupy = 0
for i in range(len(lines_exp)):
if lines_exp[i][0] == b'########\n':
# 8x #'s means match 0 or more whole lines
line_exp = lines_exp[i + 1]
skip = 0
while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]):
skip += 1
if i_mupy + skip >= len(lines_mupy):
lines_mupy[i_mupy] = b'######## FAIL\n'
break
del lines_mupy[i_mupy:i_mupy + skip]
lines_mupy.insert(i_mupy, b'########\n')
i_mupy += 1
else:
# a regex
if lines_exp[i][1].match(lines_mupy[i_mupy]):
lines_mupy[i_mupy] = lines_exp[i][0]
else:
#print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
pass
i_mupy += 1
if i_mupy >= len(lines_mupy):
break
output_mupy = b''.join(lines_mupy)
2014-10-05 17:27:12 -04:00
return output_mupy
2017-04-14 10:07:13 -04:00
def run_feature_check(pyb, args, base_path, test_file):
2020-05-09 02:20:40 -04:00
if pyb is not None and test_file.startswith("repl_"):
# REPL feature tests will not run via pyboard because they require prompt interactivity
return b""
2020-04-08 02:01:58 -04:00
return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True)
2017-04-14 10:07:13 -04:00
2020-04-08 02:01:58 -04:00
def run_tests(pyb, tests, args):
2014-05-03 11:43:27 -04:00
test_count = 0
testcase_count = 0
passed_count = 0
failed_tests = []
2014-05-10 09:52:58 -04:00
skipped_tests = []
2014-05-03 11:43:27 -04:00
2014-06-05 17:45:55 -04:00
skip_tests = set()
2014-12-12 17:50:54 -05:00
skip_native = False
2017-03-03 12:14:02 -05:00
skip_int_big = False
2019-10-18 04:48:15 -04:00
skip_bytearray = False
2017-01-04 16:16:29 -05:00
skip_set_type = False
2019-10-29 06:44:22 -04:00
skip_slice = False
2017-02-14 14:03:25 -05:00
skip_async = False
2017-04-02 15:51:31 -04:00
skip_const = False
2017-09-10 10:05:31 -04:00
skip_revops = False
2019-10-29 07:13:38 -04:00
skip_io_module = False
2017-12-15 05:07:09 -05:00
skip_endian = False
has_complex = True
has_coverage = False
upy_float_precision = 32
# If we're asked to --list-tests, we can't assume that there's a
# connection to target, so we can't run feature checks usefully.
if not (args.list_tests or args.write_exp):
2020-04-08 02:01:58 -04:00
# Even if we run completely different tests in a different directory,
# we need to access feature_checks from the same directory as the
# run-tests script itself so use base_path.
2017-12-15 05:07:09 -05:00
# Check if micropython.native is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'native_check.py')
if output == b'CRASH':
skip_native = True
# Check if arbitrary-precision integers are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'int_big.py')
if output != b'1000000000000000000000000000000000000000000000\n':
skip_int_big = True
2019-10-18 04:48:15 -04:00
# Check if bytearray is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'bytearray.py')
if output != b'bytearray\n':
skip_bytearray = True
2017-12-15 05:07:09 -05:00
# Check if set type (and set literals) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'set_check.py')
if output == b'CRASH':
skip_set_type = True
2019-10-29 06:44:22 -04:00
# Check if slice is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'slice.py')
if output != b'slice\n':
skip_slice = True
2017-12-15 05:07:09 -05:00
# Check if async/await keywords are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'async_check.py')
if output == b'CRASH':
skip_async = True
# Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'const.py')
if output == b'CRASH':
skip_const = True
# Check if __rOP__ special methods are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'reverse_ops.py')
if output == b'TypeError\n':
skip_revops = True
2019-10-29 07:13:38 -04:00
# Check if uio module exists, and skip such tests if it doesn't
output = run_feature_check(pyb, args, base_path, 'uio_module.py')
if output != b'uio\n':
skip_io_module = True
2017-12-15 05:07:09 -05:00
# Check if emacs repl is supported, and skip such tests if it's not
t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py')
2019-12-14 16:42:03 -05:00
if 'True' not in str(t, 'ascii'):
2017-12-15 05:07:09 -05:00
skip_tests.add('cmdline/repl_emacs_keys.py')
2019-12-14 16:42:03 -05:00
# Check if words movement in repl is supported, and skip such tests if it's not
t = run_feature_check(pyb, args, base_path, 'repl_words_move_check.py')
if 'True' not in str(t, 'ascii'):
skip_tests.add('cmdline/repl_words_move.py')
2017-12-15 05:07:09 -05:00
upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py')
2020-01-02 10:40:54 -05:00
upy_float_precision = run_feature_check(pyb, args, base_path, 'float.py')
if upy_float_precision == b'CRASH':
upy_float_precision = 0
else:
upy_float_precision = int(upy_float_precision)
2017-12-15 05:07:09 -05:00
has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n'
has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n'
2020-04-08 02:01:58 -04:00
cpy_byteorder = subprocess.check_output([CPYTHON3, base_path('feature_check/byteorder.py')])
2017-12-15 05:07:09 -05:00
skip_endian = (upy_byteorder != cpy_byteorder)
2015-08-29 17:28:59 -04:00
2019-10-29 07:01:17 -04:00
# These tests don't test slice explicitly but rather use it to perform the test
misc_slice_tests = (
'builtin_range',
'class_super',
'containment',
'errno1',
'fun_str',
'generator1',
'globals_del',
'memoryview1',
'memoryview_gc',
'object1',
'python34',
'struct_endian',
)
2014-06-05 17:45:55 -04:00
# Some tests shouldn't be run under Travis CI
if os.getenv('TRAVIS') == 'true':
skip_tests.add('basics/memoryerror.py')
2016-08-27 09:41:16 -04:00
skip_tests.add('thread/thread_gc1.py') # has reliability issues
2016-08-16 23:48:09 -04:00
skip_tests.add('thread/thread_lock4.py') # has reliability issues
2016-08-25 00:06:40 -04:00
skip_tests.add('thread/stress_heap.py') # has reliability issues
2016-09-07 23:06:29 -04:00
skip_tests.add('thread/stress_recurse.py') # has reliability issues
2014-05-03 11:43:27 -04:00
2017-06-25 23:47:00 -04:00
if upy_float_precision == 0:
2018-07-22 10:19:22 -04:00
skip_tests.add('extmod/uctypes_le_float.py')
skip_tests.add('extmod/uctypes_native_float.py')
skip_tests.add('extmod/uctypes_sizeof_float.py')
2017-06-25 23:47:00 -04:00
skip_tests.add('extmod/ujson_dumps_float.py')
skip_tests.add('extmod/ujson_loads_float.py')
2018-07-22 10:19:22 -04:00
skip_tests.add('extmod/urandom_extra_float.py')
2017-06-25 23:47:00 -04:00
skip_tests.add('misc/rge_sm.py')
if upy_float_precision < 32:
skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead
skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead
skip_tests.add('float/bytes_construct.py') # requires fp32
skip_tests.add('float/bytearray_construct.py') # requires fp32
if upy_float_precision < 64:
skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead
skip_tests.add('float/float2int_doubleprec_intbig.py')
2017-11-26 20:51:52 -05:00
skip_tests.add('float/float_parse_doubleprec.py')
2017-06-25 23:47:00 -04:00
2015-12-06 07:56:37 -05:00
if not has_complex:
skip_tests.add('float/complex1.py')
2017-04-03 22:26:43 -04:00
skip_tests.add('float/complex1_intbig.py')
2020-06-21 20:21:02 -04:00
skip_tests.add('float/complex_special_mehods.py')
2015-12-06 07:56:37 -05:00
skip_tests.add('float/int_big_float.py')
skip_tests.add('float/true_value.py')
skip_tests.add('float/types.py')
2016-12-21 19:26:06 -05:00
if not has_coverage:
skip_tests.add('cmdline/cmd_parsetree.py')
2016-07-09 11:52:57 -04:00
# Some tests shouldn't be run on a PC
2017-12-14 06:36:06 -05:00
if args.target == 'unix':
2016-07-09 11:52:57 -04:00
# unix build does not have the GIL so can't run thread mutation tests
for t in tests:
if t.startswith('thread/mutate_'):
skip_tests.add(t)
2014-09-13 13:43:09 -04:00
# Some tests shouldn't be run on pyboard
2017-12-14 06:36:06 -05:00
if args.target != 'unix':
2015-10-20 18:55:50 -04:00
skip_tests.add('basics/exception_chain.py') # warning is not printed
2015-04-04 17:05:30 -04:00
skip_tests.add('micropython/meminfo.py') # output is very different to PC output
2015-10-20 18:55:50 -04:00
skip_tests.add('extmod/machine_mem.py') # raw memory access not supported
2014-09-13 13:43:09 -04:00
2015-06-23 15:24:51 -04:00
if args.target == 'wipy':
skip_tests.add('misc/print_exception.py') # requires error reporting full
2016-02-21 16:30:35 -05:00
skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes
2015-06-23 15:24:51 -04:00
skip_tests.add('extmod/zlibd_decompress.py') # requires zlib
2015-09-27 12:47:35 -04:00
skip_tests.add('extmod/uheapq1.py') # uheapq not supported by WiPy
2016-02-21 16:30:35 -05:00
skip_tests.add('extmod/urandom_basic.py') # requires urandom
skip_tests.add('extmod/urandom_extra.py') # requires urandom
2016-02-13 11:28:02 -05:00
elif args.target == 'esp8266':
skip_tests.add('misc/rge_sm.py') # too large
2017-04-03 03:20:41 -04:00
elif args.target == 'minimal':
2017-09-10 15:32:08 -04:00
skip_tests.add('basics/class_inplace_op.py') # all special methods not supported
2017-12-14 03:32:34 -05:00
skip_tests.add('basics/subclass_native_init.py')# native subclassing corner cases not support
2017-04-03 03:20:41 -04:00
skip_tests.add('misc/rge_sm.py') # too large
skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored
2018-07-19 18:50:00 -04:00
elif args.target == 'nrf':
skip_tests.add('basics/memoryview1.py') # no item assignment for memoryview
skip_tests.add('extmod/urandom_basic.py') # unimplemented: urandom.seed
skip_tests.add('micropython/opt_level.py') # no support for line numbers
skip_tests.add('misc/non_compliant.py') # no item assignment for bytearray
for t in tests:
if t.startswith('basics/io_'):
skip_tests.add(t)
2019-08-18 20:59:27 -04:00
elif args.target == 'qemu-arm':
skip_tests.add('misc/print_exception.py') # requires sys stdfiles
2015-06-23 15:24:51 -04:00
2014-09-17 17:56:34 -04:00
# Some tests are known to fail on 64-bit machines
if pyb is None and platform.architecture()[0] == '64bit':
2014-09-23 08:09:26 -04:00
pass
2014-09-17 17:56:34 -04:00
2014-10-04 02:39:15 -04:00
# Some tests use unsupported features on Windows
if os.name == 'nt':
2015-08-06 06:31:37 -04:00
skip_tests.add('import/import_file.py') # works but CPython prints forward slashes
2014-10-04 02:39:15 -04:00
2014-08-29 14:47:10 -04:00
# Some tests are known to fail with native emitter
# Remove them from the below when they work
if args.emit == 'native':
2018-09-30 23:10:38 -04:00
skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from_close generator_name'.split()}) # require raise_varargs, generator name
skip_tests.update({'basics/async_%s.py' % t for t in 'with with2 with_break with_return'.split()}) # require async_with
2015-04-06 17:38:53 -04:00
skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs
2020-06-16 08:49:38 -04:00
skip_tests.add('basics/annotate_var.py') # requires checking for unbound local
2015-04-06 17:38:53 -04:00
skip_tests.add('basics/del_deref.py') # requires checking for unbound local
skip_tests.add('basics/del_local.py') # requires checking for unbound local
2015-09-01 04:53:27 -04:00
skip_tests.add('basics/exception_chain.py') # raise from is not supported
2018-10-26 01:48:07 -04:00
skip_tests.add('basics/scope_implicit.py') # requires checking for unbound local
2018-09-04 00:33:43 -04:00
skip_tests.add('basics/try_finally_return2.py') # requires raise_varargs
2015-04-06 17:38:53 -04:00
skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local
2020-01-23 01:57:46 -05:00
skip_tests.add('extmod/uasyncio_lock.py') # requires async with
2015-04-06 17:38:53 -04:00
skip_tests.add('misc/features.py') # requires raise_varargs
2014-12-11 12:34:55 -05:00
skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info
2015-04-24 20:17:41 -04:00
skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native
2017-09-21 01:24:57 -04:00
skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info
2016-11-21 01:39:23 -05:00
skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info
2019-08-22 21:25:46 -04:00
skip_tests.add('micropython/opt_level_lineno.py') # native doesn't have proper traceback info
2017-03-16 03:05:00 -04:00
skip_tests.add('micropython/schedule.py') # native code doesn't check pending events
2014-08-29 14:47:10 -04:00
2014-05-03 11:43:27 -04:00
for test_file in tests:
2015-08-06 06:31:37 -04:00
test_file = test_file.replace('\\', '/')
2017-12-14 05:26:10 -05:00
if args.filters:
# Default verdict is the opposit of the first action
verdict = "include" if args.filters[0][0] == "exclude" else "exclude"
for action, pat in args.filters:
if pat.search(test_file):
verdict = action
if verdict == "exclude":
continue
2018-08-08 09:20:22 -04:00
test_basename = test_file.replace('..', '_').replace('./', '').replace('/', '_')
test_name = os.path.splitext(os.path.basename(test_file))[0]
2020-03-11 05:01:46 -04:00
is_native = test_name.startswith("native_") or test_name.startswith("viper_") or args.emit == "native"
2015-08-29 17:28:59 -04:00
is_endian = test_name.endswith("_endian")
2017-03-03 16:13:27 -05:00
is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
2019-10-18 04:48:15 -04:00
is_bytearray = test_name.startswith("bytearray") or test_name.endswith("_bytearray")
2017-01-30 16:25:09 -05:00
is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")
2019-10-29 07:01:17 -04:00
is_slice = test_name.find("slice") != -1 or test_name in misc_slice_tests
2020-01-23 01:57:46 -05:00
is_async = test_name.startswith(("async_", "uasyncio_"))
2017-04-02 15:51:31 -04:00
is_const = test_name.startswith("const")
2019-10-29 07:13:38 -04:00
is_io_module = test_name.startswith("io_")
2017-01-04 16:16:29 -05:00
skip_it = test_file in skip_tests
skip_it |= skip_native and is_native
skip_it |= skip_endian and is_endian
2017-03-03 12:14:02 -05:00
skip_it |= skip_int_big and is_int_big
2019-10-18 04:48:15 -04:00
skip_it |= skip_bytearray and is_bytearray
2017-01-04 16:16:29 -05:00
skip_it |= skip_set_type and is_set_type
2019-10-29 06:44:22 -04:00
skip_it |= skip_slice and is_slice
2017-02-14 14:03:25 -05:00
skip_it |= skip_async and is_async
2017-04-02 15:51:31 -04:00
skip_it |= skip_const and is_const
2017-09-10 10:05:31 -04:00
skip_it |= skip_revops and test_name.startswith("class_reverse_op")
2019-10-29 07:13:38 -04:00
skip_it |= skip_io_module and is_io_module
2014-06-05 17:41:30 -04:00
2017-12-14 05:11:17 -05:00
if args.list_tests:
if not skip_it:
print(test_file)
continue
2017-01-04 16:16:29 -05:00
if skip_it:
2014-05-03 11:43:27 -04:00
print("skip ", test_file)
2014-05-10 09:52:58 -04:00
skipped_tests.append(test_name)
2014-05-03 11:43:27 -04:00
continue
# get expected output
test_file_expected = test_file + '.exp'
if os.path.isfile(test_file_expected):
# expected output given by a file, so read that in
with open(test_file_expected, 'rb') as f:
output_expected = f.read()
else:
2014-05-11 06:45:02 -04:00
# run CPython to work out expected output
2014-05-03 11:43:27 -04:00
try:
output_expected = subprocess.check_output([CPYTHON3, '-B', test_file])
2014-07-12 09:34:51 -04:00
if args.write_exp:
with open(test_file_expected, 'wb') as f:
f.write(output_expected)
2014-05-03 11:43:27 -04:00
except subprocess.CalledProcessError:
output_expected = b'CPYTHON3 CRASH'
2016-03-07 07:01:21 -05:00
# canonical form for all host platforms is to use \n for end-of-line
output_expected = output_expected.replace(b'\r\n', b'\n')
2014-07-12 09:34:51 -04:00
if args.write_exp:
continue
2017-06-30 03:22:17 -04:00
# run MicroPython
2014-10-05 17:27:12 -04:00
output_mupy = run_micropython(pyb, args, test_file)
2014-05-03 11:43:27 -04:00
2016-03-07 07:01:21 -05:00
if output_mupy == b'SKIP\n':
2014-05-10 09:52:58 -04:00
print("skip ", test_file)
skipped_tests.append(test_name)
continue
testcase_count += len(output_expected.splitlines())
2020-04-08 02:01:58 -04:00
filename_expected = base_path(test_basename + ".exp")
filename_mupy = base_path(test_basename + ".out")
2014-05-03 11:43:27 -04:00
if output_expected == output_mupy:
print("pass ", test_file)
passed_count += 1
rm_f(filename_expected)
rm_f(filename_mupy)
else:
2014-06-28 05:29:12 -04:00
with open(filename_expected, "wb") as f:
f.write(output_expected)
with open(filename_mupy, "wb") as f:
f.write(output_mupy)
2014-05-03 11:43:27 -04:00
print("FAIL ", test_file)
failed_tests.append(test_name)
test_count += 1
2017-12-13 11:01:35 -05:00
if args.list_tests:
return True
2014-05-03 11:43:27 -04:00
print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
print("{} tests passed".format(passed_count))
2014-05-10 09:52:58 -04:00
if len(skipped_tests) > 0:
print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests)))
2014-05-03 11:43:27 -04:00
if len(failed_tests) > 0:
print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests)))
return False
# all tests succeeded
return True
2017-12-14 05:26:10 -05:00
class append_filter(argparse.Action):
def __init__(self, option_strings, dest, **kwargs):
super().__init__(option_strings, dest, default=[], **kwargs)
def __call__(self, parser, args, value, option):
if not hasattr(args, self.dest):
args.filters = []
if option.startswith(("-e", "--e")):
option = "exclude"
else:
option = "include"
args.filters.append((option, re.compile(value)))
2014-05-03 11:43:27 -04:00
def main():
2017-12-14 05:26:10 -05:00
cmd_parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
2020-01-16 12:07:50 -05:00
description='''Run and manage tests for MicroPython.
2020-04-08 02:01:58 -04:00
Tests are discovered by scanning test directories for .py files or using the
specified test files. If test files nor directories are specified, the script
expects to be ran in the tests directory (where this file is located) and the
builtin tests suitable for the target platform are ran.
2020-01-16 12:07:50 -05:00
When running tests, run-tests compares the MicroPython output of the test with the output
produced by running the test through CPython unless a <test>.exp file is found, in which
case it is used as comparison.
If a test fails, run-tests produces a pair of <test>.out and <test>.exp files in the current
directory with the MicroPython output and the expectations, respectively.
''',
2017-12-14 05:26:10 -05:00
epilog='''\
Options -i and -e can be multiple and processed in the order given. Regex
"search" (vs "match") operation is used. An action (include/exclude) of
the last matching regex is used:
2020-03-26 23:41:38 -04:00
run-tests -i async - exclude all, then include tests containing "async" anywhere
2017-12-14 05:26:10 -05:00
run-tests -e '/big.+int' - include all, then exclude by regex
run-tests -e async -i async_foo - include all, exclude async, yet still include async_foo
''')
2015-06-23 15:24:51 -04:00
cmd_parser.add_argument('--target', default='unix', help='the target platform')
2015-06-28 08:01:27 -04:00
cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
2014-05-31 13:11:01 -04:00
cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
2017-12-14 05:26:10 -05:00
cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py')
cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py')
2020-01-16 12:07:50 -05:00
cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython')
2017-12-13 11:01:35 -05:00
cmd_parser.add_argument('--list-tests', action='store_true', help='list tests instead of running them')
2015-10-11 19:06:25 -04:00
cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)')
2016-03-15 09:04:43 -04:00
cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')
2016-09-19 22:19:35 -04:00
cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')
2019-12-22 07:51:57 -05:00
cmd_parser.add_argument('--mpy-cross-flags', default='-mcache-lookup-bc', help='flags to pass to mpy-cross')
2015-03-24 06:55:26 -04:00
cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')
2014-05-03 11:43:27 -04:00
cmd_parser.add_argument('files', nargs='*', help='input test files')
2020-03-26 23:41:38 -04:00
cmd_parser.add_argument('--print-failures', action='store_true', help='print the diff of expected vs. actual output for failed tests and exit')
cmd_parser.add_argument('--clean-failures', action='store_true', help='delete the .exp and .out files from failed tests and exit')
2014-05-03 11:43:27 -04:00
args = cmd_parser.parse_args()
2020-03-26 23:41:38 -04:00
if args.print_failures:
2020-04-08 02:01:58 -04:00
for exp in glob(base_path("*.exp")):
testbase = exp[:-4]
2020-03-26 23:41:38 -04:00
print()
print("FAILURE {0}".format(testbase))
2020-03-30 04:14:09 -04:00
os.system("{0} {1}.exp {1}.out".format(DIFF, testbase))
2020-03-26 23:41:38 -04:00
sys.exit(0)
if args.clean_failures:
2020-04-08 02:01:58 -04:00
for f in glob(base_path("*.exp")) + glob(base_path("*.out")):
2020-03-26 23:41:38 -04:00
os.remove(f)
sys.exit(0)
2019-08-18 20:59:27 -04:00
LOCAL_TARGETS = ('unix', 'qemu-arm',)
2018-07-19 18:50:00 -04:00
EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'esp32', 'minimal', 'nrf')
2019-08-18 20:59:27 -04:00
if args.target in LOCAL_TARGETS or args.list_tests:
2017-12-13 11:01:35 -05:00
pyb = None
elif args.target in EXTERNAL_TARGETS:
2018-09-05 01:35:18 -04:00
global pyboard
2020-04-08 02:01:58 -04:00
sys.path.append(base_path('../tools'))
2014-05-03 11:43:27 -04:00
import pyboard
2015-06-28 08:01:27 -04:00
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
2014-04-13 08:48:33 -04:00
pyb.enter_raw_repl()
2015-06-23 15:24:51 -04:00
else:
2019-08-18 20:59:27 -04:00
raise ValueError('target must be one of %s' % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS))
2014-05-03 11:43:27 -04:00
if len(args.files) == 0:
2014-05-28 08:27:54 -04:00
if args.test_dirs is None:
2019-08-18 20:59:27 -04:00
test_dirs = ('basics', 'micropython', 'misc', 'extmod',)
2015-06-23 15:24:51 -04:00
if args.target == 'pyboard':
2014-05-28 08:27:54 -04:00
# run pyboard tests
2019-08-18 20:59:27 -04:00
test_dirs += ('float', 'stress', 'pyb', 'pybnative', 'inlineasm')
2018-07-19 18:50:00 -04:00
elif args.target in ('esp8266', 'esp32', 'minimal', 'nrf'):
2019-08-18 20:59:27 -04:00
test_dirs += ('float',)
2015-06-23 15:24:51 -04:00
elif args.target == 'wipy':
# run WiPy tests
2019-08-18 20:59:27 -04:00
test_dirs += ('wipy',)
elif args.target == 'unix':
2015-06-23 15:24:51 -04:00
# run PC tests
2019-08-18 20:59:27 -04:00
test_dirs += ('float', 'import', 'io', 'stress', 'unicode', 'unix', 'cmdline',)
elif args.target == 'qemu-arm':
if not args.write_exp:
raise ValueError('--target=qemu-arm must be used with --write-exp')
# Generate expected output files for qemu run.
# This list should match the test_dirs tuple in tinytest-codegen.py.
test_dirs += ('float', 'inlineasm', 'qemu-arm',)
2014-05-03 11:43:27 -04:00
else:
2014-05-28 08:27:54 -04:00
# run tests from these directories
test_dirs = args.test_dirs
2014-05-03 11:43:27 -04:00
tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
2014-01-27 16:53:28 -05:00
else:
2014-05-03 11:43:27 -04:00
# tests explicitly given
tests = args.files
2014-01-27 16:53:28 -05:00
2015-03-24 06:55:26 -04:00
if not args.keep_path:
2020-03-09 11:58:47 -04:00
# clear search path to make sure tests use only builtin modules and those in extmod
2020-04-08 02:01:58 -04:00
os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod')
2015-03-24 06:55:26 -04:00
2017-10-07 08:49:58 -04:00
try:
2020-04-08 02:01:58 -04:00
res = run_tests(pyb, tests, args)
2017-10-07 08:49:58 -04:00
finally:
if pyb:
pyb.close()
2017-04-02 13:47:44 -04:00
if not res:
2014-05-03 11:43:27 -04:00
sys.exit(1)
2014-01-27 16:53:28 -05:00
2014-05-03 11:43:27 -04:00
if __name__ == "__main__":
main()