py/objgenerator: Implement PEP479, StopIteration convs to RuntimeError.

This commit implements PEP479 which disallows raising StopIteration inside
a generator to signal that it should be finished.  Instead, the generator
should simply return when it is complete.

See https://www.python.org/dev/peps/pep-0479/ for details.

py/objgenerator.c

@@ -145,6 +145,10 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
             size_t n_state = mp_decode_uint_value(self->code_state.fun_bc->bytecode);
             self->code_state.ip = 0;
             *ret_val = self->code_state.state[n_state - 1];
+            // PEP479: if StopIteration is raised inside a generator it is replaced with RuntimeError
+            if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(*ret_val)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
+                *ret_val = mp_obj_new_exception_msg(&mp_type_RuntimeError, "generator raised StopIteration");
+            }
             break;
         }
     }
@@ -168,15 +172,6 @@ STATIC mp_obj_t gen_resume_and_raise(mp_obj_t self_in, mp_obj_t send_value, mp_o
             return ret;
 
         case MP_VM_RETURN_EXCEPTION:
-            // TODO: Optimization of returning MP_OBJ_STOP_ITERATION is really part
-            // of mp_iternext() protocol, but this function is called by other methods
-            // too, which may not handled MP_OBJ_STOP_ITERATION.
-            if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
-                mp_obj_t val = mp_obj_exception_get_value(ret);
-                if (val == mp_const_none) {
-                    return MP_OBJ_STOP_ITERATION;
-                }
-            }
             nlr_raise(ret);
     }
 }
@@ -216,11 +211,10 @@ STATIC mp_obj_t gen_instance_close(mp_obj_t self_in) {
         case MP_VM_RETURN_YIELD:
             mp_raise_msg(&mp_type_RuntimeError, "generator ignored GeneratorExit");
 
-        // Swallow StopIteration & GeneratorExit (== successful close), and re-raise any other
+        // Swallow GeneratorExit (== successful close), and re-raise any other
         case MP_VM_RETURN_EXCEPTION:
             // ret should always be an instance of an exception class
-            if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit)) ||
-                mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
+            if (mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(mp_obj_get_type(ret)), MP_OBJ_FROM_PTR(&mp_type_GeneratorExit))) {
                 return mp_const_none;
             }
             nlr_raise(ret);

tests/basics/gen_yield_from.py

@@ -13,34 +13,6 @@ g = gen2()
 print(list(g))
 
 
-# Like above, but terminate subgen using StopIteration
-def gen3():
-    yield 1
-    yield 2
-    raise StopIteration
-
-def gen4():
-    print("here1")
-    print((yield from gen3()))
-    print("here2")
-
-g = gen4()
-print(list(g))
-
-# Like above, but terminate subgen using StopIteration with value
-def gen5():
-    yield 1
-    yield 2
-    raise StopIteration(123)
-
-def gen6():
-    print("here1")
-    print((yield from gen5()))
-    print("here2")
-
-g = gen6()
-print(list(g))
-
 # StopIteration from within a Python function, within a native iterator (map), within a yield from
 def gen7(x):
     if x < 3:

tests/basics/gen_yield_from.py.exp

@@ -1,14 +0,0 @@
-here1
-3
-here2
-[1, 2]
-here1
-None
-here2
-[1, 2]
-here1
-123
-here2
-[1, 2]
-444
-[0, 1, 2]

tests/basics/gen_yield_from_close.py

@@ -55,14 +55,14 @@ except StopIteration:
 
 
 # Yet another variation - leaf generator gets GeneratorExit,
-# but raises StopIteration instead. This still should close chain properly.
+# and reraises a new GeneratorExit. This still should close chain properly.
 def gen5():
     yield 1
     try:
         yield 2
     except GeneratorExit:
-        print("leaf caught GeneratorExit and raised StopIteration instead")
-        raise StopIteration(123)
+        print("leaf caught GeneratorExit and reraised GeneratorExit")
+        raise GeneratorExit(123)
     yield 3
     yield 4
 

tests/basics/gen_yield_from_close.py.exp

@@ -1,20 +0,0 @@
--1
-1
-StopIteration
--1
-1
-2
-leaf caught GeneratorExit and swallowed it
-delegating caught GeneratorExit
-StopIteration
--1
-1
-2
-leaf caught GeneratorExit and raised StopIteration instead
-delegating caught GeneratorExit
-StopIteration
-123
-RuntimeError
-0
-1
-close

tests/basics/gen_yield_from_throw.py

@@ -25,6 +25,20 @@ def gen3():
 g3 = gen3()
 print(next(g3))
 try:
-    g3.throw(StopIteration)
+    g3.throw(KeyError)
+except KeyError:
+    print('got KeyError from downstream!')
+
+# case where a thrown exception is caught and stops the generator
+def gen4():
+    try:
+        yield 1
+        yield 2
+    except:
+        pass
+g4 = gen4()
+print(next(g4))
+try:
+    g4.throw(ValueError)
 except StopIteration:
-    print('got StopIteration from downstream!')
+    print('got StopIteration')

tests/basics/gen_yield_from_throw.py.exp

@@ -1,6 +0,0 @@
-1
-got ValueError from upstream!
-str1
-got TypeError from downstream!
-123
-got StopIteration from downstream!

tests/basics/generator_close.py

@@ -31,13 +31,14 @@ except StopIteration:
     print("StopIteration")
 
 
-# Throwing StopIteration in response to close() is ok
+# Throwing GeneratorExit in response to close() is ok
 def gen2():
     try:
         yield 1
         yield 2
     except:
-        raise StopIteration
+        print('raising GeneratorExit')
+        raise GeneratorExit
 
 g = gen2()
 next(g)

tests/basics/generator_close.py.exp

@@ -1,10 +0,0 @@
-None
-StopIteration
-1
-None
-StopIteration
-[1, 2]
-None
-StopIteration
-None
-ValueError

tests/basics/generator_pep479.py

@@ -0,0 +1,29 @@
+# tests for correct PEP479 behaviour (introduced in Python 3.5)
+
+# basic case: StopIteration is converted into a RuntimeError
+def gen():
+    yield 1
+    raise StopIteration
+g = gen()
+print(next(g))
+try:
+    next(g)
+except RuntimeError:
+    print('RuntimeError')
+
+# trying to continue a failed generator now raises StopIteration
+try:
+    next(g)
+except StopIteration:
+    print('StopIteration')
+
+# throwing a StopIteration which is uncaught will be converted into a RuntimeError
+def gen():
+    yield 1
+    yield 2
+g = gen()
+print(next(g))
+try:
+    g.throw(StopIteration)
+except RuntimeError:
+    print('RuntimeError')

tests/basics/generator_pep479.py.exp

@@ -0,0 +1,5 @@
+1
+RuntimeError
+StopIteration
+1
+RuntimeError

tests/run-tests

@@ -352,7 +352,7 @@ def run_tests(pyb, tests, args, base_path="."):
     # Some tests are known to fail with native emitter
     # Remove them from the below when they work
     if args.emit == 'native':
-        skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_executing gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_name generator_pend_throw generator_return generator_send'.split()}) # require yield
+        skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_executing gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_name generator_pend_throw generator_return generator_send generator_pep479'.split()}) # require yield
         skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield
         skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2 with_break with_return'.split()}) # require yield
         skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs