2014-04-02 10:31:39 -04:00
#! /usr/bin/env python3
2014-01-27 16:53:28 -05:00
import os
import subprocess
import sys
2014-09-17 17:56:34 -04:00
import platform
2014-05-03 11:43:27 -04:00
import argparse
2015-03-20 13:25:25 -04:00
import re
2018-03-31 10:15:29 -04:00
import threading
2018-03-31 10:38:25 -04:00
import multiprocessing
2018-03-31 10:32:20 -04:00
from multiprocessing.pool import ThreadPool
2014-01-27 16:53:28 -05:00
from glob import glob
2014-04-04 10:28:34 -04:00
# Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
2014-01-27 16:53:28 -05:00
if os.name == 'nt':
2014-04-04 10:28:34 -04:00
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
2017-09-06 00:09:13 -04:00
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe')
2014-01-27 16:53:28 -05:00
else:
2014-04-03 15:06:35 -04:00
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
2017-09-06 00:09:13 -04:00
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython')
2014-01-27 16:53:28 -05:00
2016-09-19 22:19:35 -04:00
# mpy-cross is only needed if --via-mpy command-line arg is passed
MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross')
2014-11-19 10:44:31 -05:00
# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
2014-12-11 17:51:39 -05:00
os.environ['PYTHONIOENCODING'] = 'utf-8'
2014-11-19 10:44:31 -05:00
2014-03-22 19:07:30 -04:00
def rm_f(fname):
if os.path.exists(fname):
os.remove(fname)
2016-07-26 14:29:48 -04:00
# unescape wanted regex chars and escape unwanted ones
def convert_regex_escapes(line):
cs = []
escape = False
for c in str(line, 'utf8'):
if escape:
escape = False
cs.append(c)
elif c == '\\':
escape = True
elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'):
cs.append('\\' + c)
else:
cs.append(c)
# accept carriage-return(s) before final newline
if cs[-1] == '\n':
cs[-1] = '\r*\n'
return bytes(''.join(cs), 'utf8')
2017-04-14 10:07:13 -04:00
def run_micropython(pyb, args, test_file, is_special=False):
2017-12-04 18:52:41 -05:00
special_tests = (
'micropython/meminfo.py', 'basics/bytes_compare3.py',
'basics/builtin_help.py', 'thread/thread_exc2.py',
)
2018-02-14 01:19:44 -05:00
had_crash = False
2014-10-05 17:27:12 -04:00
if pyb is None:
# run on PC
2016-07-26 14:29:48 -04:00
if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests:
2015-03-13 06:58:34 -04:00
# special handling for tests of the unix cmdline program
2016-07-26 14:29:48 -04:00
is_special = True
2015-03-13 06:58:34 -04:00
2017-04-14 10:07:13 -04:00
if is_special:
2015-03-13 06:58:34 -04:00
# check for any cmdline options needed for this test
args = [MICROPYTHON]
with open(test_file, 'rb') as f:
line = f.readline()
if line.startswith(b'# cmdline:'):
2015-08-06 06:34:48 -04:00
# subprocess.check_output on Windows only accepts strings, not bytes
args += [str(c, 'utf-8') for c in line[10:].strip().split()]
2015-03-13 06:58:34 -04:00
# run the test, possibly with redirected input
try:
2015-08-30 04:36:42 -04:00
if 'repl_' in test_file:
2015-07-23 15:05:30 -04:00
# Need to use a PTY to test command line editing
2015-08-06 06:34:48 -04:00
try:
import pty
except ImportError:
# in case pty module is not available, like on Windows
return b'SKIP\n'
2015-07-23 15:05:30 -04:00
import select
2015-11-24 12:52:03 -05:00
def get(required=False):
2015-07-23 15:05:30 -04:00
rv = b''
while True:
2020-06-14 15:28:32 -04:00
ready = select.select([emulator], [], [], 0.02)
if ready[0] == [emulator]:
rv += os.read(emulator, 1024)
2015-07-23 15:05:30 -04:00
else:
2015-11-24 12:52:03 -05:00
if not required or rv:
return rv
2015-07-23 15:05:30 -04:00
def send_get(what):
2020-06-14 15:28:32 -04:00
os.write(emulator, what)
2015-07-23 15:05:30 -04:00
return get()
with open(test_file, 'rb') as f:
# instead of: output_mupy = subprocess.check_output(args, stdin=f)
2020-06-14 15:28:32 -04:00
# openpty returns two read/write file descriptors. The first one is
# used by the program which provides the virtual
# terminal service, and the second one is used by the
# subprogram which requires a tty to work.
emulator, subterminal = pty.openpty()
p = subprocess.Popen(args, stdin=subterminal, stdout=subterminal,
2015-07-23 15:05:30 -04:00
stderr=subprocess.STDOUT, bufsize=0)
2015-11-24 12:52:03 -05:00
banner = get(True)
2015-07-23 15:05:30 -04:00
output_mupy = banner + b''.join(send_get(line) for line in f)
2016-12-19 23:13:49 -05:00
send_get(b'\x04') # exit the REPL, so coverage info is saved
2015-07-23 15:05:30 -04:00
p.kill()
2020-06-14 15:28:32 -04:00
os.close(emulator)
os.close(subterminal)
2015-03-13 06:58:34 -04:00
else:
2017-09-24 21:15:48 -04:00
output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT)
2015-03-13 06:58:34 -04:00
except subprocess.CalledProcessError:
2015-09-21 06:24:17 -04:00
return b'CRASH'
2015-03-13 06:58:34 -04:00
else:
2016-09-19 22:19:35 -04:00
# a standard test run on PC
# create system command
cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit]
if args.heapsize is not None:
cmdlist.extend(['-X', 'heapsize=' + args.heapsize])
# if running via .mpy, first compile the .py file
if args.via_mpy:
subprocess.check_output([MPYCROSS, '-mcache-lookup-bc', '-o', 'mpytest.mpy', test_file])
cmdlist.extend(['-m', 'mpytest'])
else:
2016-03-15 09:04:43 -04:00
cmdlist.append(test_file)
2016-09-19 22:19:35 -04:00
# run the actual test
2017-07-03 18:05:08 -04:00
e = {"MICROPYPATH": os.getcwd() + ":", "LANG": "en_US.UTF-8"}
2017-10-25 14:43:02 -04:00
p = subprocess.Popen(cmdlist, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
2017-07-03 18:05:08 -04:00
output_mupy = b''
while p.poll() is None:
output_mupy += p.stdout.read()
output_mupy += p.stdout.read()
if p.returncode != 0:
2015-03-13 06:58:34 -04:00
output_mupy = b'CRASH'
2016-09-19 22:19:35 -04:00
# clean up if we had an intermediate .mpy file
if args.via_mpy:
rm_f('mpytest.mpy')
2014-10-05 17:27:12 -04:00
else:
# run on pyboard
import pyboard
pyb.enter_raw_repl()
try:
2016-03-07 07:01:21 -05:00
output_mupy = pyb.execfile(test_file)
2014-10-05 17:27:12 -04:00
except pyboard.PyboardError:
2018-02-14 01:19:44 -05:00
had_crash = True
2014-10-05 17:27:12 -04:00
output_mupy = b'CRASH'
2016-03-07 07:01:21 -05:00
# canonical form for all ports/platforms is to use \n for end-of-line
output_mupy = output_mupy.replace(b'\r\n', b'\n')
2017-01-23 04:44:29 -05:00
# don't try to convert the output if we should skip this test
2018-02-14 01:19:44 -05:00
if had_crash or output_mupy in (b'SKIP\n', b'CRASH'):
2017-01-23 04:44:29 -05:00
return output_mupy
2016-07-26 14:29:48 -04:00
if is_special or test_file in special_tests:
# convert parts of the output that are not stable across runs
with open(test_file + '.exp', 'rb') as f:
lines_exp = []
for line in f.readlines():
if line == b'########\n':
line = (line,)
else:
line = (line, re.compile(convert_regex_escapes(line)))
lines_exp.append(line)
lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]
if output_mupy.endswith(b'\n'):
lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
i_mupy = 0
for i in range(len(lines_exp)):
if lines_exp[i][0] == b'########\n':
# 8x #'s means match 0 or more whole lines
line_exp = lines_exp[i + 1]
skip = 0
while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]):
skip += 1
if i_mupy + skip >= len(lines_mupy):
lines_mupy[i_mupy] = b'######## FAIL\n'
break
del lines_mupy[i_mupy:i_mupy + skip]
lines_mupy.insert(i_mupy, b'########\n')
i_mupy += 1
else:
# a regex
if lines_exp[i][1].match(lines_mupy[i_mupy]):
lines_mupy[i_mupy] = lines_exp[i][0]
else:
#print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
pass
i_mupy += 1
if i_mupy >= len(lines_mupy):
break
output_mupy = b''.join(lines_mupy)
2014-10-05 17:27:12 -04:00
return output_mupy
2017-04-14 10:07:13 -04:00
def run_feature_check(pyb, args, base_path, test_file):
return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True)
2018-03-31 10:15:29 -04:00
class ThreadSafeCounter:
def __init__(self, start=0):
self._value = start
self._lock = threading.Lock()
def add(self, to_add):
with self._lock: self._value += to_add
def append(self, arg):
self.add([arg])
@property
def value(self):
return self._value
2017-04-14 10:07:13 -04:00
2018-03-31 10:32:20 -04:00
def run_tests(pyb, tests, args, base_path=".", num_threads=1):
2018-03-31 10:15:29 -04:00
test_count = ThreadSafeCounter()
testcase_count = ThreadSafeCounter()
passed_count = ThreadSafeCounter()
failed_tests = ThreadSafeCounter([])
skipped_tests = ThreadSafeCounter([])
2014-05-03 11:43:27 -04:00
2014-06-05 17:45:55 -04:00
skip_tests = set()
2014-12-12 17:50:54 -05:00
skip_native = False
2017-03-03 12:14:02 -05:00
skip_int_big = False
2017-01-04 16:16:29 -05:00
skip_set_type = False
2017-02-14 14:03:25 -05:00
skip_async = False
2017-04-02 15:51:31 -04:00
skip_const = False
2017-09-10 10:05:31 -04:00
skip_revops = False
2017-12-15 05:07:09 -05:00
skip_endian = False
has_complex = True
has_coverage = False
upy_float_precision = 32
# If we're asked to --list-tests, we can't assume that there's a
# connection to target, so we can't run feature checks usefully.
if not (args.list_tests or args.write_exp):
# Check if micropython.native is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'native_check.py')
if output == b'CRASH':
skip_native = True
# Check if arbitrary-precision integers are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'int_big.py')
if output != b'1000000000000000000000000000000000000000000000\n':
skip_int_big = True
# Check if set type (and set literals) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'set_check.py')
if output == b'CRASH':
skip_set_type = True
# Check if async/await keywords are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'async_check.py')
if output == b'CRASH':
skip_async = True
# Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'const.py')
if output == b'CRASH':
skip_const = True
# Check if __rOP__ special methods are supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'reverse_ops.py')
if output == b'TypeError\n':
skip_revops = True
# Check if emacs repl is supported, and skip such tests if it's not
t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py')
if not 'True' in str(t, 'ascii'):
skip_tests.add('cmdline/repl_emacs_keys.py')
upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py')
upy_float_precision = int(run_feature_check(pyb, args, base_path, 'float.py'))
has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n'
has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n'
cpy_byteorder = subprocess.check_output([CPYTHON3, base_path + '/feature_check/byteorder.py'])
skip_endian = (upy_byteorder != cpy_byteorder)
2015-08-29 17:28:59 -04:00
2014-06-05 17:45:55 -04:00
# Some tests shouldn't be run under Travis CI
if os.getenv('TRAVIS') == 'true':
skip_tests.add('basics/memoryerror.py')
2016-08-27 09:41:16 -04:00
skip_tests.add('thread/thread_gc1.py') # has reliability issues
2016-08-16 23:48:09 -04:00
skip_tests.add('thread/thread_lock4.py') # has reliability issues
2016-08-25 00:06:40 -04:00
skip_tests.add('thread/stress_heap.py') # has reliability issues
2016-09-07 23:06:29 -04:00
skip_tests.add('thread/stress_recurse.py') # has reliability issues
2014-05-03 11:43:27 -04:00
2017-06-25 23:47:00 -04:00
if upy_float_precision == 0:
skip_tests.add('extmod/ujson_dumps_float.py')
skip_tests.add('extmod/ujson_loads_float.py')
skip_tests.add('misc/rge_sm.py')
if upy_float_precision < 32:
skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead
skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead
skip_tests.add('float/bytes_construct.py') # requires fp32
skip_tests.add('float/bytearray_construct.py') # requires fp32
if upy_float_precision < 64:
skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead
skip_tests.add('float/float2int_doubleprec_intbig.py')
2017-11-26 20:51:52 -05:00
skip_tests.add('float/float_parse_doubleprec.py')
2017-06-25 23:47:00 -04:00
2015-12-06 07:56:37 -05:00
if not has_complex:
skip_tests.add('float/complex1.py')
2017-04-03 22:26:43 -04:00
skip_tests.add('float/complex1_intbig.py')
2015-12-06 07:56:37 -05:00
skip_tests.add('float/int_big_float.py')
skip_tests.add('float/true_value.py')
skip_tests.add('float/types.py')
2016-12-21 19:26:06 -05:00
if not has_coverage:
skip_tests.add('cmdline/cmd_parsetree.py')
2016-07-09 11:52:57 -04:00
# Some tests shouldn't be run on a PC
2017-12-14 06:36:06 -05:00
if args.target == 'unix':
2016-07-09 11:52:57 -04:00
# unix build does not have the GIL so can't run thread mutation tests
for t in tests:
if t.startswith('thread/mutate_'):
skip_tests.add(t)
2014-09-13 13:43:09 -04:00
# Some tests shouldn't be run on pyboard
2017-12-14 06:36:06 -05:00
if args.target != 'unix':
2015-10-20 18:55:50 -04:00
skip_tests.add('basics/exception_chain.py') # warning is not printed
2015-04-04 17:05:30 -04:00
skip_tests.add('micropython/meminfo.py') # output is very different to PC output
2015-10-20 18:55:50 -04:00
skip_tests.add('extmod/machine_mem.py') # raw memory access not supported
2014-09-13 13:43:09 -04:00
2015-06-23 15:24:51 -04:00
if args.target == 'wipy':
skip_tests.add('misc/print_exception.py') # requires error reporting full
2016-02-21 16:30:35 -05:00
skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes
2015-06-23 15:24:51 -04:00
skip_tests.add('extmod/zlibd_decompress.py') # requires zlib
2015-09-27 12:47:35 -04:00
skip_tests.add('extmod/uheapq1.py') # uheapq not supported by WiPy
2016-02-21 16:30:35 -05:00
skip_tests.add('extmod/urandom_basic.py') # requires urandom
skip_tests.add('extmod/urandom_extra.py') # requires urandom
2016-02-13 11:28:02 -05:00
elif args.target == 'esp8266':
skip_tests.add('misc/rge_sm.py') # too large
2017-04-03 03:20:41 -04:00
elif args.target == 'minimal':
2017-09-10 15:32:08 -04:00
skip_tests.add('basics/class_inplace_op.py') # all special methods not supported
2017-12-14 03:32:34 -05:00
skip_tests.add('basics/subclass_native_init.py')# native subclassing corner cases not support
2017-04-03 03:20:41 -04:00
skip_tests.add('misc/rge_sm.py') # too large
skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored
2015-06-23 15:24:51 -04:00
2014-09-17 17:56:34 -04:00
# Some tests are known to fail on 64-bit machines
if pyb is None and platform.architecture()[0] == '64bit':
2014-09-23 08:09:26 -04:00
pass
2014-09-17 17:56:34 -04:00
2014-10-04 02:39:15 -04:00
# Some tests use unsupported features on Windows
if os.name == 'nt':
2015-08-06 06:31:37 -04:00
skip_tests.add('import/import_file.py') # works but CPython prints forward slashes
2014-10-04 02:39:15 -04:00
2014-08-29 14:47:10 -04:00
# Some tests are known to fail with native emitter
# Remove them from the below when they work
if args.emit == 'native':
2018-05-22 02:54:03 -04:00
skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_executing gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_pend_throw generator_return generator_send'.split()}) # require yield
2018-04-01 13:15:35 -04:00
skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join gen_stack_overflow'.split()}) # require yield
2016-10-10 21:30:32 -04:00
skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield
2015-04-06 17:38:53 -04:00
skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs
2016-04-07 03:53:24 -04:00
skip_tests.update({'basics/%s.py' % t for t in 'with_break with_continue with_return'.split()}) # require complete with support
2015-03-25 19:33:48 -04:00
skip_tests.add('basics/array_construct2.py') # requires generators
skip_tests.add('basics/bool1.py') # seems to randomly fail
2017-07-06 21:47:38 -04:00
skip_tests.add('basics/builtin_hash_gen.py') # requires yield
2015-12-26 08:06:50 -05:00
skip_tests.add('basics/class_bind_self.py') # requires yield
2015-04-06 17:38:53 -04:00
skip_tests.add('basics/del_deref.py') # requires checking for unbound local
skip_tests.add('basics/del_local.py') # requires checking for unbound local
2015-09-01 04:53:27 -04:00
skip_tests.add('basics/exception_chain.py') # raise from is not supported
2015-12-08 16:39:21 -05:00
skip_tests.add('basics/for_range.py') # requires yield_value
2015-04-06 17:38:53 -04:00
skip_tests.add('basics/try_finally_loops.py') # requires proper try finally code
skip_tests.add('basics/try_finally_return.py') # requires proper try finally code
2015-12-24 07:49:31 -05:00
skip_tests.add('basics/try_finally_return2.py') # requires proper try finally code
2015-04-06 17:38:53 -04:00
skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local
skip_tests.add('import/gen_context.py') # requires yield_value
skip_tests.add('misc/features.py') # requires raise_varargs
skip_tests.add('misc/rge_sm.py') # requires yield
2014-12-11 12:34:55 -05:00
skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info
2015-04-24 20:17:41 -04:00
skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native
2017-09-21 01:24:57 -04:00
skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info
2016-11-21 01:39:23 -05:00
skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info
2017-02-10 00:09:20 -05:00
skip_tests.add('micropython/heapalloc_iter.py') # requires generators
2017-03-16 03:05:00 -04:00
skip_tests.add('micropython/schedule.py') # native code doesn't check pending events
2018-04-03 23:58:57 -04:00
skip_tests.add('stress/gc_trace.py') # requires yield
2018-04-01 13:15:35 -04:00
skip_tests.add('stress/recursive_gen.py') # requires yield
2018-06-04 02:25:06 -04:00
skip_tests.add('extmod/vfs_userfs.py') # because native doesn't properly handle globals across different modules
2020-04-13 17:18:42 -04:00
skip_tests.add('../extmod/ulab/tests/argminmax.py') # requires yield
2014-08-29 14:47:10 -04:00
2018-03-31 10:08:37 -04:00
def run_one_test(test_file):
2015-08-06 06:31:37 -04:00
test_file = test_file.replace('\\', '/')
2017-12-14 05:26:10 -05:00
if args.filters:
# Default verdict is the opposit of the first action
verdict = "include" if args.filters[0][0] == "exclude" else "exclude"
for action, pat in args.filters:
if pat.search(test_file):
verdict = action
if verdict == "exclude":
2018-07-28 13:29:47 -04:00
return
2017-12-14 05:26:10 -05:00
2014-06-05 17:41:30 -04:00
test_basename = os.path.basename(test_file)
test_name = os.path.splitext(test_basename)[0]
2014-12-12 17:50:54 -05:00
is_native = test_name.startswith("native_") or test_name.startswith("viper_")
2015-08-29 17:28:59 -04:00
is_endian = test_name.endswith("_endian")
2017-03-03 16:13:27 -05:00
is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
2017-01-30 16:25:09 -05:00
is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")
2017-02-14 14:03:25 -05:00
is_async = test_name.startswith("async_")
2017-04-02 15:51:31 -04:00
is_const = test_name.startswith("const")
2017-01-04 16:16:29 -05:00
skip_it = test_file in skip_tests
skip_it |= skip_native and is_native
skip_it |= skip_endian and is_endian
2017-03-03 12:14:02 -05:00
skip_it |= skip_int_big and is_int_big
2017-01-04 16:16:29 -05:00
skip_it |= skip_set_type and is_set_type
2017-02-14 14:03:25 -05:00
skip_it |= skip_async and is_async
2017-04-02 15:51:31 -04:00
skip_it |= skip_const and is_const
2017-09-10 10:05:31 -04:00
skip_it |= skip_revops and test_name.startswith("class_reverse_op")
2014-06-05 17:41:30 -04:00
2017-12-14 05:11:17 -05:00
if args.list_tests:
if not skip_it:
print(test_file)
2018-07-28 13:29:47 -04:00
return
2017-12-14 05:11:17 -05:00
2017-01-04 16:16:29 -05:00
if skip_it:
2014-05-03 11:43:27 -04:00
print("skip ", test_file)
2014-05-10 09:52:58 -04:00
skipped_tests.append(test_name)
2018-03-31 10:08:37 -04:00
return
2014-05-03 11:43:27 -04:00
# get expected output
test_file_expected = test_file + '.exp'
if os.path.isfile(test_file_expected):
# expected output given by a file, so read that in
with open(test_file_expected, 'rb') as f:
output_expected = f.read()
else:
2014-05-11 06:45:02 -04:00
# run CPython to work out expected output
2017-07-03 18:05:08 -04:00
e = {"PYTHONPATH": os.getcwd(),
"PATH": os.environ["PATH"],
"LANG": "en_US.UTF-8"}
p = subprocess.Popen([CPYTHON3, '-B', test_file], env=e, stdout=subprocess.PIPE)
output_expected = b''
while p.poll() is None:
output_expected += p.stdout.read()
output_expected += p.stdout.read()
if p.returncode != 0:
2014-05-03 11:43:27 -04:00
output_expected = b'CPYTHON3 CRASH'
2017-07-03 18:05:08 -04:00
elif args.write_exp:
with open(test_file_expected, 'wb') as f:
f.write(output_expected)
2014-05-03 11:43:27 -04:00
2016-03-07 07:01:21 -05:00
# canonical form for all host platforms is to use \n for end-of-line
output_expected = output_expected.replace(b'\r\n', b'\n')
2014-07-12 09:34:51 -04:00
if args.write_exp:
2018-03-31 10:08:37 -04:00
return
2014-07-12 09:34:51 -04:00
2017-06-30 03:22:17 -04:00
# run MicroPython
2014-10-05 17:27:12 -04:00
output_mupy = run_micropython(pyb, args, test_file)
2014-05-03 11:43:27 -04:00
2016-03-07 07:01:21 -05:00
if output_mupy == b'SKIP\n':
2014-05-10 09:52:58 -04:00
print("skip ", test_file)
skipped_tests.append(test_name)
2018-03-31 10:08:37 -04:00
return
2014-05-10 09:52:58 -04:00
2018-03-31 10:15:29 -04:00
testcase_count.add(len(output_expected.splitlines()))
2014-05-10 09:52:58 -04:00
2014-05-03 11:43:27 -04:00
filename_expected = test_basename + ".exp"
filename_mupy = test_basename + ".out"
if output_expected == output_mupy:
print("pass ", test_file)
2018-03-31 10:15:29 -04:00
passed_count.add(1)
2014-05-03 11:43:27 -04:00
rm_f(filename_expected)
rm_f(filename_mupy)
else:
2014-06-28 05:29:12 -04:00
with open(filename_expected, "wb") as f:
f.write(output_expected)
with open(filename_mupy, "wb") as f:
f.write(output_mupy)
2017-07-03 18:05:08 -04:00
print("### Expected")
print(output_expected)
print("### Actual")
print(output_mupy)
2014-05-03 11:43:27 -04:00
print("FAIL ", test_file)
failed_tests.append(test_name)
2018-03-31 10:15:29 -04:00
test_count.add(1)
2014-05-03 11:43:27 -04:00
2017-12-13 11:01:35 -05:00
if args.list_tests:
return True
2018-03-31 10:32:20 -04:00
if num_threads > 1:
pool = ThreadPool(num_threads)
pool.map(run_one_test, tests)
else:
for test in tests:
run_one_test(test)
2018-03-31 10:08:37 -04:00
2018-03-31 10:15:29 -04:00
print("{} tests performed ({} individual testcases)".format(test_count.value, testcase_count.value))
print("{} tests passed".format(passed_count.value))
2014-05-03 11:43:27 -04:00
2018-03-31 10:15:29 -04:00
if len(skipped_tests.value) > 0:
2018-03-31 10:34:01 -04:00
print("{} tests skipped: {}".format(len(skipped_tests.value), ' '.join(sorted(skipped_tests.value))))
2018-03-31 10:15:29 -04:00
if len(failed_tests.value) > 0:
2018-03-31 10:34:01 -04:00
print("{} tests failed: {}".format(len(failed_tests.value), ' '.join(sorted(failed_tests.value))))
2014-05-03 11:43:27 -04:00
return False
# all tests succeeded
return True
2017-12-14 05:26:10 -05:00
class append_filter(argparse.Action):
def __init__(self, option_strings, dest, **kwargs):
super().__init__(option_strings, dest, default=[], **kwargs)
def __call__(self, parser, args, value, option):
if not hasattr(args, self.dest):
args.filters = []
if option.startswith(("-e", "--e")):
option = "exclude"
else:
option = "include"
args.filters.append((option, re.compile(value)))
2014-05-03 11:43:27 -04:00
def main():
2017-12-14 05:26:10 -05:00
cmd_parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description='Run and manage tests for MicroPython.',
epilog='''\
Options -i and -e can be multiple and processed in the order given. Regex
"search" (vs "match") operation is used. An action (include/exclude) of
the last matching regex is used:
run-tests -i async - exclude all, then include tests containg "async" anywhere
run-tests -e '/big.+int' - include all, then exclude by regex
run-tests -e async -i async_foo - include all, exclude async, yet still include async_foo
''')
2015-06-23 15:24:51 -04:00
cmd_parser.add_argument('--target', default='unix', help='the target platform')
2015-06-28 08:01:27 -04:00
cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
2014-05-31 13:11:01 -04:00
cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
2017-12-14 05:26:10 -05:00
cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py')
cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py')
2014-07-12 09:34:51 -04:00
cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython')
2017-12-13 11:01:35 -05:00
cmd_parser.add_argument('--list-tests', action='store_true', help='list tests instead of running them')
2015-10-11 19:06:25 -04:00
cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)')
2016-03-15 09:04:43 -04:00
cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')
2016-09-19 22:19:35 -04:00
cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')
2015-03-24 06:55:26 -04:00
cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')
2018-03-31 10:32:20 -04:00
cmd_parser.add_argument('-j', '--jobs', default=1, metavar='N', type=int, help='Number of tests to run simultaneously')
2018-03-31 10:38:25 -04:00
cmd_parser.add_argument('--auto-jobs', action='store_const', dest='jobs', const=multiprocessing.cpu_count(), help='Set the -j values to the CPU (thread) count')
2014-05-03 11:43:27 -04:00
cmd_parser.add_argument('files', nargs='*', help='input test files')
args = cmd_parser.parse_args()
2018-01-01 23:55:22 -05:00
EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'esp32', 'minimal')
2017-12-13 11:01:35 -05:00
if args.target == 'unix' or args.list_tests:
pyb = None
elif args.target in EXTERNAL_TARGETS:
2014-05-03 11:43:27 -04:00
import pyboard
2015-06-28 08:01:27 -04:00
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
2014-04-13 08:48:33 -04:00
pyb.enter_raw_repl()
2015-06-23 15:24:51 -04:00
else:
2016-05-07 11:45:16 -04:00
raise ValueError('target must be either %s or unix' % ", ".join(EXTERNAL_TARGETS))
2014-05-03 11:43:27 -04:00
if len(args.files) == 0:
2014-05-28 08:27:54 -04:00
if args.test_dirs is None:
2015-06-23 15:24:51 -04:00
if args.target == 'pyboard':
2014-05-28 08:27:54 -04:00
# run pyboard tests
2016-03-15 09:20:18 -04:00
test_dirs = ('basics', 'micropython', 'float', 'misc', 'stress', 'extmod', 'pyb', 'pybnative', 'inlineasm')
2018-01-01 23:55:22 -05:00
elif args.target in ('esp8266', 'esp32', 'minimal'):
2016-02-13 10:01:27 -05:00
test_dirs = ('basics', 'micropython', 'float', 'misc', 'extmod')
2015-06-23 15:24:51 -04:00
elif args.target == 'wipy':
# run WiPy tests
2015-09-03 05:25:44 -04:00
test_dirs = ('basics', 'micropython', 'misc', 'extmod', 'wipy')
2015-06-23 15:24:51 -04:00
else:
# run PC tests
2017-12-04 18:52:41 -05:00
test_dirs = (
'basics', 'micropython', 'float', 'import', 'io', 'misc',
2020-02-27 15:17:29 -05:00
'stress', 'unicode', 'extmod', '../extmod/ulab/tests', 'unix', 'cmdline',
2017-12-04 18:52:41 -05:00
)
2014-05-03 11:43:27 -04:00
else:
2014-05-28 08:27:54 -04:00
# run tests from these directories
test_dirs = args.test_dirs
2014-05-03 11:43:27 -04:00
tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
2014-01-27 16:53:28 -05:00
else:
2014-05-03 11:43:27 -04:00
# tests explicitly given
tests = args.files
2014-01-27 16:53:28 -05:00
2015-03-24 06:55:26 -04:00
if not args.keep_path:
# clear search path to make sure tests use only builtin modules
os.environ['MICROPYPATH'] = ''
2017-04-14 10:07:13 -04:00
# Even if we run completely different tests in a different directory,
# we need to access feature_check's from the same directory as the
# run-tests script itself.
base_path = os.path.dirname(sys.argv[0]) or "."
2017-10-07 08:49:58 -04:00
try:
2018-03-31 10:32:20 -04:00
res = run_tests(pyb, tests, args, base_path, args.jobs)
2017-10-07 08:49:58 -04:00
finally:
if pyb:
pyb.close()
2017-04-02 13:47:44 -04:00
if not res:
2014-05-03 11:43:27 -04:00
sys.exit(1)
2014-01-27 16:53:28 -05:00
2014-05-03 11:43:27 -04:00
if __name__ == "__main__":
main()