djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #729 from jepler/tests-parallel-circuitpython

Browse Source 
Optionally parallelize the testsuite
This commit is contained in:
Scott Shawcroft 2018-03-31 09:55:21 -07:00 committed by GitHub
commit d65ea992bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.travis.yml
View File

@ -78,24 +78,24 @@ script:
- echo -en 'travis_fold:end:qemu\\r' - echo -en 'travis_fold:end:qemu\\r'
# run tests without coverage info # run tests without coverage info
#- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests) #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --auto-jobs)
#- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --emit native) #- (cd tests && MICROPY_CPYTHON3=python3.4 ./run-tests --auto-jobs --emit native)
# run tests with coverage info # run tests with coverage info
- echo 'Test all' && echo -en 'travis_fold:start:test_all\\r' - echo 'Test all' && echo -en 'travis_fold:start:test_all\\r'
- ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests)) - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs))
- echo -en 'travis_fold:end:test_all\\r' - echo -en 'travis_fold:end:test_all\\r'
- echo 'Test threads' && echo -en 'travis_fold:start:test_threads\\r' - echo 'Test threads' && echo -en 'travis_fold:start:test_threads\\r'
- ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -d thread)) - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs -d thread))
- echo -en 'travis_fold:end:test_threads\\r' - echo -en 'travis_fold:end:test_threads\\r'
- echo 'Testing with native' && echo -en 'travis_fold:start:test_native\\r' - echo 'Testing with native' && echo -en 'travis_fold:start:test_native\\r'
- ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --emit native)) - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --auto-jobs --emit native))
- echo -en 'travis_fold:end:test_native\\r' - echo -en 'travis_fold:end:test_native\\r'
- (echo 'Testing with mpy' && echo -en 'travis_fold:start:test_mpy\\r') - (echo 'Testing with mpy' && echo -en 'travis_fold:start:test_mpy\\r')
- ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests --via-mpy -d basics float)) - ([[ $TRAVIS_TEST != "unix" ]] || (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../ports/unix/micropython_coverage ./run-tests -j1 --via-mpy -d basics float))
- echo -en 'travis_fold:end:test_mpy\\r' - echo -en 'travis_fold:end:test_mpy\\r'
- (echo 'Building docs' && echo -en 'travis_fold:start:build_docs\\r') - (echo 'Building docs' && echo -en 'travis_fold:start:build_docs\\r')

10
ports/unix/Makefile
View File

@ -189,7 +189,7 @@ include $(TOP)/py/mkrules.mk
test: $(PROG) $(TOP)/tests/run-tests test: $(PROG) $(TOP)/tests/run-tests
$(eval DIRNAME=ports/$(notdir $(CURDIR))) $(eval DIRNAME=ports/$(notdir $(CURDIR)))
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests --auto-jobs
# install micropython in /usr/local/bin # install micropython in /usr/local/bin
TARGET = micropython TARGET = micropython
@ -254,10 +254,10 @@ coverage:
coverage_test: coverage coverage_test: coverage
$(eval DIRNAME=ports/$(notdir $(CURDIR))) $(eval DIRNAME=ports/$(notdir $(CURDIR)))
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests -d thread cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs -d thread
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --emit native cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --auto-jobs --emit native
cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests --via-mpy -d basics float cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/micropython_coverage ./run-tests -j1 --via-mpy -d basics float
gcov -o build-coverage/py $(TOP)/py/*.c gcov -o build-coverage/py $(TOP)/py/*.c
gcov -o build-coverage/extmod $(TOP)/extmod/*.c gcov -o build-coverage/extmod $(TOP)/extmod/*.c

2
ports/windows/.appveyor.yml
View File

@ -22,7 +22,7 @@ test_script:
- cmd: >- - cmd: >-
cd tests cd tests
%MICROPY_CPYTHON3% run-tests %MICROPY_CPYTHON3% run-tests --auto-jobs
skip_tags: true skip_tags: true

67
tests/run-tests
View File

@ -6,6 +6,9 @@ import sys
import platform import platform
import argparse import argparse
import re import re
import threading
import multiprocessing
from multiprocessing.pool import ThreadPool
from glob import glob from glob import glob
# Tests require at least CPython 3.3. If your default python3 executable # Tests require at least CPython 3.3. If your default python3 executable
@ -197,13 +200,27 @@ def run_micropython(pyb, args, test_file, is_special=False):
def run_feature_check(pyb, args, base_path, test_file): def run_feature_check(pyb, args, base_path, test_file):
return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True) return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True)
class ThreadSafeCounter:
def __init__(self, start=0):
self._value = start
self._lock = threading.Lock()
def run_tests(pyb, tests, args, base_path="."): def add(self, to_add):
test_count = 0 with self._lock: self._value += to_add
testcase_count = 0
passed_count = 0 def append(self, arg):
failed_tests = [] self.add([arg])
skipped_tests = []
@property
def value(self):
return self._value
def run_tests(pyb, tests, args, base_path=".", num_threads=1):
test_count = ThreadSafeCounter()
testcase_count = ThreadSafeCounter()
passed_count = ThreadSafeCounter()
failed_tests = ThreadSafeCounter([])
skipped_tests = ThreadSafeCounter([])
skip_tests = set() skip_tests = set()
skip_native = False skip_native = False
@ -354,7 +371,7 @@ def run_tests(pyb, tests, args, base_path="."):
skip_tests.add('micropython/heapalloc_iter.py') # requires generators skip_tests.add('micropython/heapalloc_iter.py') # requires generators
skip_tests.add('micropython/schedule.py') # native code doesn't check pending events skip_tests.add('micropython/schedule.py') # native code doesn't check pending events
for test_file in tests: def run_one_test(test_file):
test_file = test_file.replace('\\', '/') test_file = test_file.replace('\\', '/')
test_basename = os.path.basename(test_file) test_basename = os.path.basename(test_file)
test_name = os.path.splitext(test_basename)[0] test_name = os.path.splitext(test_basename)[0]
@ -377,7 +394,7 @@ def run_tests(pyb, tests, args, base_path="."):
if skip_it: if skip_it:
print("skip ", test_file) print("skip ", test_file)
skipped_tests.append(test_name) skipped_tests.append(test_name)
continue return
# get expected output # get expected output
test_file_expected = test_file + '.exp' test_file_expected = test_file + '.exp'
@ -405,7 +422,7 @@ def run_tests(pyb, tests, args, base_path="."):
output_expected = output_expected.replace(b'\r\n', b'\n') output_expected = output_expected.replace(b'\r\n', b'\n')
if args.write_exp: if args.write_exp:
continue return
# run MicroPython # run MicroPython
output_mupy = run_micropython(pyb, args, test_file) output_mupy = run_micropython(pyb, args, test_file)
@ -413,16 +430,16 @@ def run_tests(pyb, tests, args, base_path="."):
if output_mupy == b'SKIP\n': if output_mupy == b'SKIP\n':
print("skip ", test_file) print("skip ", test_file)
skipped_tests.append(test_name) skipped_tests.append(test_name)
continue return
testcase_count += len(output_expected.splitlines()) testcase_count.add(len(output_expected.splitlines()))
filename_expected = test_basename + ".exp" filename_expected = test_basename + ".exp"
filename_mupy = test_basename + ".out" filename_mupy = test_basename + ".out"
if output_expected == output_mupy: if output_expected == output_mupy:
print("pass ", test_file) print("pass ", test_file)
passed_count += 1 passed_count.add(1)
rm_f(filename_expected) rm_f(filename_expected)
rm_f(filename_mupy) rm_f(filename_mupy)
else: else:
@ -437,15 +454,22 @@ def run_tests(pyb, tests, args, base_path="."):
print("FAIL ", test_file) print("FAIL ", test_file)
failed_tests.append(test_name) failed_tests.append(test_name)
test_count += 1 test_count.add(1)
print("{} tests performed ({} individual testcases)".format(test_count, testcase_count)) if num_threads > 1:
print("{} tests passed".format(passed_count)) pool = ThreadPool(num_threads)
pool.map(run_one_test, tests)
else:
for test in tests:
run_one_test(test)
if len(skipped_tests) > 0: print("{} tests performed ({} individual testcases)".format(test_count.value, testcase_count.value))
print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests))) print("{} tests passed".format(passed_count.value))
if len(failed_tests) > 0:
print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests))) if len(skipped_tests.value) > 0:
print("{} tests skipped: {}".format(len(skipped_tests.value), ' '.join(sorted(skipped_tests.value))))
if len(failed_tests.value) > 0:
print("{} tests failed: {}".format(len(failed_tests.value), ' '.join(sorted(failed_tests.value))))
return False return False
# all tests succeeded # all tests succeeded
@ -464,11 +488,14 @@ def main():
cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')
cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')
cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')
cmd_parser.add_argument('-j', '--jobs', default=1, metavar='N', type=int, help='Number of tests to run simultaneously')
cmd_parser.add_argument('--auto-jobs', action='store_const', dest='jobs', const=multiprocessing.cpu_count(), help='Set the -j values to the CPU (thread) count')
cmd_parser.add_argument('files', nargs='*', help='input test files') cmd_parser.add_argument('files', nargs='*', help='input test files')
args = cmd_parser.parse_args() args = cmd_parser.parse_args()
EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal') EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal')
if args.target in EXTERNAL_TARGETS: if args.target in EXTERNAL_TARGETS:
args.jobs = 1
import pyboard import pyboard
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
pyb.enter_raw_repl() pyb.enter_raw_repl()
@ -507,7 +534,7 @@ def main():
# run-tests script itself. # run-tests script itself.
base_path = os.path.dirname(sys.argv[0]) or "." base_path = os.path.dirname(sys.argv[0]) or "."
try: try:
res = run_tests(pyb, tests, args, base_path) res = run_tests(pyb, tests, args, base_path, args.jobs)
finally: finally:
if pyb: if pyb:
pyb.close() pyb.close()