Merge pull request #7059 from jepler/asyncio-tests-dogfood

asyncio: we should dogfood our own asyncio implementation during automated tests

extmod/moduasyncio.c

@@ -70,24 +70,13 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf);
 /******************************************************************************/
 // Ticks for task ordering in pairing heap
 
-#if !CIRCUITPY || (defined(__unix__) || defined(__APPLE__))
-STATIC mp_obj_t ticks(void) {
-    return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1));
-}
-
-STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) {
-    mp_uint_t t0 = MP_OBJ_SMALL_INT_VALUE(t0_in);
-    mp_uint_t t1 = MP_OBJ_SMALL_INT_VALUE(t1_in);
-    mp_int_t diff = ((t1 - t0 + MICROPY_PY_UTIME_TICKS_PERIOD / 2) & (MICROPY_PY_UTIME_TICKS_PERIOD - 1))
-        - MICROPY_PY_UTIME_TICKS_PERIOD / 2;
-    return diff;
-}
-#else
 #define _TICKS_PERIOD (1lu << 29)
 #define _TICKS_MAX (_TICKS_PERIOD - 1)
 #define _TICKS_HALFPERIOD (_TICKS_PERIOD >> 1)
 
-#define ticks() supervisor_ticks_ms()
+STATIC mp_obj_t ticks(void) {
+    return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & _TICKS_MAX);
+}
 
 STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) {
     mp_uint_t t0 = MP_OBJ_SMALL_INT_VALUE(t0_in);
@@ -95,7 +84,6 @@ STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) {
     mp_int_t diff = ((t1 - t0 + _TICKS_HALFPERIOD) & _TICKS_MAX) - _TICKS_HALFPERIOD;
     return diff;
 }
-#endif
 
 STATIC int task_lt(mp_pairheap_t *n1, mp_pairheap_t *n2) {
     mp_obj_task_t *t1 = (mp_obj_task_t *)n1;
@@ -302,8 +290,13 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) {
 STATIC mp_obj_t task_iternext(mp_obj_t self_in) {
     mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in);
     if (TASK_IS_DONE(self)) {
-        // Task finished, raise return value to caller so it can continue.
-        nlr_raise(self->data);
+        if (self->data == mp_const_none) {
+            // Task finished but has already been sent to the loop's exception handler.
+            mp_raise_StopIteration(MP_OBJ_NULL);
+        } else {
+            // Task finished, raise return value to caller so it can continue.
+            nlr_raise(self->data);
+        }
     } else {
         // Put calling task on waiting queue.
         mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task));

extmod/uasyncio/task.py

@@ -141,8 +141,12 @@ class Task:
 
     def __next__(self):
         if not self.state:
-            # Task finished, raise return value to caller so it can continue.
-            raise self.data
+            if self.data is None:
+                # Task finished but has already been sent to the loop's exception handler.
+                raise StopIteration
+            else:
+                # Task finished, raise return value to caller so it can continue.
+                raise self.data
         else:
             # Put calling task on waiting queue.
             self.state.push_head(core.cur_task)

frozen/Adafruit_CircuitPython_Display_Shapes

@@ -1 +1 @@
-Subproject commit 288a4f553d2f616331b5a042568c97b5bb0e44a7
+Subproject commit 656be4d79196b5f25ab9ebca731d448c5b3bdc17

frozen/Adafruit_CircuitPython_Display_Text

@@ -1 +1 @@
-Subproject commit 41f06c33ef7a029210416ac61319698f5768e83e
+Subproject commit 6cf9f3cf32e0c176c861de6356813ea4d08034d6

frozen/Adafruit_CircuitPython_RFM69

@@ -1 +1 @@
-Subproject commit a815f364badc0dac3fe49e7d8206d00ce95894e1
+Subproject commit 96b4a05c8a225ad7ddc392be7fb69efebe151981

frozen/Adafruit_CircuitPython_SD

@@ -1 +1 @@
-Subproject commit c59df6b8c3d8006c290b63e95996e49e8e7124c0
+Subproject commit bb2fc8c157ee44869cde4cbc1ab20e6f31ac727f

frozen/Adafruit_CircuitPython_Ticks

@@ -1 +1 @@
-Subproject commit 5436a0f27f33b364035ce499c020b274d77a25b7
+Subproject commit 7832bbb5449d55d8c7b731e4ff7490b801e94a9e

frozen/Adafruit_CircuitPython_asyncio

@@ -1 +1 @@
-Subproject commit 3e8c50eb2230de7b8a20dedac33334b5dbdee21e
+Subproject commit fbdb77d7127e7d6a8d3574440b0f790c94a28cf8

ports/unix/Makefile

@@ -296,9 +296,10 @@ include $(TOP)/py/mkrules.mk
 
 .PHONY: test test_full
 
+TEST_EXTRA ?=
 test: $(PROG) $(TOP)/tests/run-tests.py
 	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
-	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py
+	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py $(TEST_EXTRA)
 
 test_full: $(PROG) $(TOP)/tests/run-tests.py
 	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
@@ -309,6 +310,10 @@ test_full: $(PROG) $(TOP)/tests/run-tests.py
 	cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) --emit native -d basics float micropython
 	cat $(TOP)/tests/basics/0prelim.py | ./$(PROG) | grep -q 'abc'
 
+.PHONY: print-failures clean-failures
+print-failures clean-failures:
+	../../tests/run-tests.py --$@
+
 test_gcov: test_full
 	gcov -o $(BUILD)/py $(TOP)/py/*.c
 	gcov -o $(BUILD)/extmod $(TOP)/extmod/*.c

tests/extmod/uasyncio_task_exception.py

@@ -0,0 +1,42 @@
+# In MicroPython, a non-awaited task with a pending exception will raise to
+# the loop's exception handler the second time it is scheduled. This is
+# because without reference counting we have no way to know when the task is
+# truly "non awaited" -- i.e. we only know that it wasn't awaited in the time
+# it took to be re-scheduled.
+
+# If the task _is_ subsequently awaited, then the await should succeed without
+# raising.
+
+try:
+    import uasyncio as asyncio
+except ImportError:
+    try:
+        import asyncio
+    except ImportError:
+        print("SKIP")
+        raise SystemExit
+
+
+def custom_handler(loop, context):
+    print("exception handler", type(context["exception"]).__name__)
+
+
+async def main():
+    loop = asyncio.get_event_loop()
+    loop.set_exception_handler(custom_handler)
+
+    async def task():
+        print("raise")
+        raise OSError
+
+    print("create")
+    t = asyncio.create_task(task())
+    print("sleep 1")
+    await asyncio.sleep(0)
+    print("sleep 2")
+    await asyncio.sleep(0)
+    print("await")
+    await t  # should not raise.
+
+
+asyncio.run(main())

tests/extmod/uasyncio_task_exception.py.exp

@@ -0,0 +1,6 @@
+create
+sleep 1
+raise
+sleep 2
+exception handler OSError
+await

tests/run-tests.py

@@ -183,7 +183,17 @@ def run_micropython(pyb, args, test_file, is_special=False):
 
             # run the actual test
             try:
-                output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT)
+                result = subprocess.run(
+                    cmdlist,
+                    stderr=subprocess.STDOUT,
+                    stdout=subprocess.PIPE,
+                    check=True,
+                    timeout=10,
+                )
+                output_mupy = result.stdout
+            except subprocess.TimeoutExpired as er:
+                had_crash = True
+                output_mupy = (er.output or b"") + b"TIMEOUT"
             except subprocess.CalledProcessError as er:
                 had_crash = True
                 output_mupy = er.output + b"CRASH"
@@ -869,9 +879,15 @@ the last matching regex is used:
         tests = args.files
 
     if not args.keep_path:
-        # clear search path to make sure tests use only builtin modules and those in extmod
-        os.environ["MICROPYPATH"] = os.pathsep + ".frozen" + os.pathsep + base_path("../extmod")
-
+        # clear search path to make sure tests use only builtin modules and those that can be frozen
+        os.environ["MICROPYPATH"] = os.pathsep.join(
+            [
+                "",
+                ".frozen",
+                base_path("../frozen/Adafruit_CircuitPython_asyncio"),
+                base_path("../frozen/Adafruit_CircuitPython_Ticks"),
+            ]
+        )
     try:
         os.makedirs(args.result_dir, exist_ok=True)
         res = run_tests(pyb, tests, args, args.result_dir, args.jobs)