From 71a3d6ec3bd02c5bd13334537e1bd146bb643bad Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Mar 2017 14:54:53 +1100 Subject: [PATCH 001/166] py: Reduce size of mp_code_state_t structure. Instead of caching data that is constant (code_info, const_table and n_state), store just a pointer to the underlying function object from which this data can be derived. This helps reduce stack usage for the case when the mp_code_state_t structure is stored on the stack, as well as heap usage when it's stored on the heap. The downside is that the VM becomes a little more complex because it now needs to derive the data from the underlying function object. But this doesn't impact the performance by much (if at all) because most of the decoding of data is done outside the main opcode loop. Measurements using pystone show that little to no performance is lost. This patch also fixes a nasty bug whereby the bytecode can be reclaimed by the GC during execution. With this patch there is always a pointer to the function object held by the VM during execution, since it's stored in the mp_code_state_t structure. --- py/bc.c | 22 +++++++++++----------- py/bc.h | 12 +++++++----- py/emitnative.c | 35 ++++++++++++----------------------- py/objfun.c | 12 ++++++------ py/objgenerator.c | 15 +++++++++------ py/vm.c | 25 +++++++++++++++++-------- 6 files changed, 62 insertions(+), 59 deletions(-) diff --git a/py/bc.c b/py/bc.c index 07de08fc36..db5d0e6869 100644 --- a/py/bc.c +++ b/py/bc.c @@ -86,25 +86,26 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // On entry code_state should be allocated somewhere (stack/heap) and // contain the following valid entries: -// - code_state->ip should contain the offset in bytes from the start of -// the bytecode chunk to just after n_state and n_exc_stack -// - code_state->n_state should be set to the state size (locals plus stack) -void mp_setup_code_state(mp_code_state_t *code_state, mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, const mp_obj_t *args) { +// - code_state->fun_bc should contain a pointer to the function object +// - code_state->ip should contain the offset in bytes from the pointer +// code_state->fun_bc->bytecode to the entry n_state (0 for bytecode, non-zero for native) +void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { // This function is pretty complicated. It's main aim is to be efficient in speed and RAM // usage for the common case of positional only args. - size_t n_state = code_state->n_state; + + // get the function object that we want to set up (could be bytecode or native code) + mp_obj_fun_bc_t *self = code_state->fun_bc; // ip comes in as an offset into bytecode, so turn it into a true pointer code_state->ip = self->bytecode + (size_t)code_state->ip; - // store pointer to constant table - code_state->const_table = self->const_table; - #if MICROPY_STACKLESS code_state->prev = NULL; #endif // get params + size_t n_state = mp_decode_uint(&code_state->ip); + mp_decode_uint(&code_state->ip); // skip n_exc_stack size_t scope_flags = *code_state->ip++; size_t n_pos_args = *code_state->ip++; size_t n_kwonly_args = *code_state->ip++; @@ -168,7 +169,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, mp_obj_fun_bc_t *self, siz } // get pointer to arg_names array - const mp_obj_t *arg_names = (const mp_obj_t*)code_state->const_table; + const mp_obj_t *arg_names = (const mp_obj_t*)self->const_table; for (size_t i = 0; i < n_kw; i++) { // the keys in kwargs are expected to be qstr objects @@ -244,9 +245,8 @@ continue2:; // get the ip and skip argument names const byte *ip = code_state->ip; - // store pointer to code_info and jump over it + // jump over code info (source file and line-number mapping) { - code_state->code_info = ip; const byte *ip2 = ip; size_t code_info_size = mp_decode_uint(&ip2); ip += code_info_size; diff --git a/py/bc.h b/py/bc.h index c7dffbac59..996b1a2f32 100644 --- a/py/bc.h +++ b/py/bc.h @@ -28,6 +28,7 @@ #include "py/runtime.h" #include "py/obj.h" +#include "py/objfun.h" // bytecode layout: // @@ -70,9 +71,12 @@ typedef struct _mp_exc_stack_t { } mp_exc_stack_t; typedef struct _mp_code_state_t { - const byte *code_info; + // The fun_bc entry points to the underlying function object that is being executed. + // It is needed to access the start of bytecode and the const_table. + // It is also needed to prevent the GC from reclaiming the bytecode during execution, + // because the ip pointer below will always point to the interior of the bytecode. + mp_obj_fun_bc_t *fun_bc; const byte *ip; - const mp_uint_t *const_table; mp_obj_t *sp; // bit 0 is saved currently_in_except_block value mp_exc_stack_t *exc_sp; @@ -80,7 +84,6 @@ typedef struct _mp_code_state_t { #if MICROPY_STACKLESS struct _mp_code_state_t *prev; #endif - size_t n_state; // Variable-length mp_obj_t state[0]; // Variable-length, never accessed by name, only as (void*)(state + n_state) @@ -91,8 +94,7 @@ mp_uint_t mp_decode_uint(const byte **ptr); mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); -struct _mp_obj_fun_bc_t; -void mp_setup_code_state(mp_code_state_t *code_state, struct _mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, const mp_obj_t *args); +void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_bytecode_print(const void *descr, const byte *code, mp_uint_t len, const mp_uint_t *const_table); void mp_bytecode_print2(const byte *code, size_t len, const mp_uint_t *const_table); const byte *mp_bytecode_print_str(const byte *ip); diff --git a/py/emitnative.c b/py/emitnative.c index 55eb6cadfc..4af68102b9 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -407,43 +407,29 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // prepare incoming arguments for call to mp_setup_code_state + #if N_X86 - asm_x86_mov_arg_to_r32(emit->as, 0, REG_ARG_2); - asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_3); - asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_4); - asm_x86_mov_arg_to_r32(emit->as, 3, REG_ARG_5); - #else - #if N_THUMB - ASM_MOV_REG_REG(emit->as, ASM_THUMB_REG_R4, REG_ARG_4); - #elif N_ARM - ASM_MOV_REG_REG(emit->as, ASM_ARM_REG_R4, REG_ARG_4); - #else - ASM_MOV_REG_REG(emit->as, REG_ARG_5, REG_ARG_4); - #endif - ASM_MOV_REG_REG(emit->as, REG_ARG_4, REG_ARG_3); - ASM_MOV_REG_REG(emit->as, REG_ARG_3, REG_ARG_2); - ASM_MOV_REG_REG(emit->as, REG_ARG_2, REG_ARG_1); + asm_x86_mov_arg_to_r32(emit->as, 0, REG_ARG_1); + asm_x86_mov_arg_to_r32(emit->as, 1, REG_ARG_2); + asm_x86_mov_arg_to_r32(emit->as, 2, REG_ARG_3); + asm_x86_mov_arg_to_r32(emit->as, 3, REG_ARG_4); #endif + // set code_state.fun_bc + ASM_MOV_REG_TO_LOCAL(emit->as, REG_ARG_1, offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)); + // set code_state.ip (offset from start of this function to prelude info) // XXX this encoding may change size - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, emit->prelude_offset, offsetof(mp_code_state_t, ip) / sizeof(mp_uint_t), REG_ARG_1); - - // set code_state.n_state - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, emit->n_state, offsetof(mp_code_state_t, n_state) / sizeof(mp_uint_t), REG_ARG_1); + ASM_MOV_IMM_TO_LOCAL_USING(emit->as, emit->prelude_offset, offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), REG_ARG_1); // put address of code_state into first arg ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, 0, REG_ARG_1); // call mp_setup_code_state to prepare code_state structure #if N_THUMB - asm_thumb_op16(emit->as, 0xb400 | (1 << ASM_THUMB_REG_R4)); // push 5th arg asm_thumb_bl_ind(emit->as, mp_fun_table[MP_F_SETUP_CODE_STATE], MP_F_SETUP_CODE_STATE, ASM_THUMB_REG_R4); - asm_thumb_op16(emit->as, 0xbc00 | (1 << REG_RET)); // pop dummy (was 5th arg) #elif N_ARM - asm_arm_push(emit->as, 1 << ASM_ARM_REG_R4); // push 5th arg asm_arm_bl_ind(emit->as, mp_fun_table[MP_F_SETUP_CODE_STATE], MP_F_SETUP_CODE_STATE, ASM_ARM_REG_R4); - asm_arm_pop(emit->as, 1 << REG_RET); // pop dummy (was 5th arg) #else ASM_CALL_IND(emit->as, mp_fun_table[MP_F_SETUP_CODE_STATE], MP_F_SETUP_CODE_STATE); #endif @@ -477,6 +463,9 @@ STATIC void emit_native_end_pass(emit_t *emit) { if (!emit->do_viper_types) { emit->prelude_offset = mp_asm_base_get_code_pos(&emit->as->base); + mp_asm_base_data(&emit->as->base, 1, 0x80 | ((emit->n_state >> 7) & 0x7f)); + mp_asm_base_data(&emit->as->base, 1, emit->n_state & 0x7f); + mp_asm_base_data(&emit->as->base, 1, 0); // n_exc_stack mp_asm_base_data(&emit->as->base, 1, emit->scope->scope_flags); mp_asm_base_data(&emit->as->base, 1, emit->scope->num_pos_args); mp_asm_base_data(&emit->as->base, 1, emit->scope->num_kwonly_args); diff --git a/py/objfun.c b/py/objfun.c index a823f49e53..4670521b41 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -220,9 +220,9 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args return NULL; } - code_state->ip = (byte*)(ip - self->bytecode); // offset to after n_state/n_exc_stack - code_state->n_state = n_state; - mp_setup_code_state(code_state, self, n_args, n_kw, args); + code_state->fun_bc = self; + code_state->ip = 0; + mp_setup_code_state(code_state, n_args, n_kw, args); // execute the byte code with the correct globals context code_state->old_globals = mp_globals_get(); @@ -265,9 +265,9 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const state_size = 0; // indicate that we allocated using alloca } - code_state->ip = (byte*)(ip - self->bytecode); // offset to after n_state/n_exc_stack - code_state->n_state = n_state; - mp_setup_code_state(code_state, self, n_args, n_kw, args); + code_state->fun_bc = self; + code_state->ip = 0; + mp_setup_code_state(code_state, n_args, n_kw, args); // execute the byte code with the correct globals context code_state->old_globals = mp_globals_get(); diff --git a/py/objgenerator.c b/py/objgenerator.c index 654b186703..2baead2315 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -67,9 +67,9 @@ STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons o->base.type = &mp_type_gen_instance; o->globals = self_fun->globals; - o->code_state.n_state = n_state; - o->code_state.ip = (byte*)(ip - self_fun->bytecode); // offset to prelude - mp_setup_code_state(&o->code_state, self_fun, n_args, n_kw, args); + o->code_state.fun_bc = self_fun; + o->code_state.ip = 0; + mp_setup_code_state(&o->code_state, n_args, n_kw, args); return MP_OBJ_FROM_PTR(o); } @@ -92,7 +92,7 @@ mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun) { STATIC void gen_instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); - mp_printf(print, "", mp_obj_code_get_name(self->code_state.code_info), self); + mp_printf(print, "", mp_obj_fun_get_name(MP_OBJ_FROM_PTR(self->code_state.fun_bc)), self); } mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { @@ -134,10 +134,13 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ } break; - case MP_VM_RETURN_EXCEPTION: + case MP_VM_RETURN_EXCEPTION: { + const byte *bc = self->code_state.fun_bc->bytecode; + size_t n_state = mp_decode_uint(&bc); self->code_state.ip = 0; - *ret_val = self->code_state.state[self->code_state.n_state - 1]; + *ret_val = self->code_state.state[n_state - 1]; break; + } } return ret_kind; diff --git a/py/vm.c b/py/vm.c index 7a906cd804..63a88a4fb5 100644 --- a/py/vm.c +++ b/py/vm.c @@ -73,10 +73,10 @@ typedef enum { ip += 2; #define DECODE_PTR \ DECODE_UINT; \ - void *ptr = (void*)(uintptr_t)code_state->const_table[unum] + void *ptr = (void*)(uintptr_t)code_state->fun_bc->const_table[unum] #define DECODE_OBJ \ DECODE_UINT; \ - mp_obj_t obj = (mp_obj_t)code_state->const_table[unum] + mp_obj_t obj = (mp_obj_t)code_state->fun_bc->const_table[unum] #else @@ -162,8 +162,10 @@ mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp run_code_state: ; #endif // Pointers which are constant for particular invocation of mp_execute_bytecode() - mp_obj_t * /*const*/ fastn = &code_state->state[code_state->n_state - 1]; - mp_exc_stack_t * /*const*/ exc_stack = (mp_exc_stack_t*)(code_state->state + code_state->n_state); + const byte *temp_bc = code_state->fun_bc->bytecode; + size_t n_state = mp_decode_uint(&temp_bc); + mp_obj_t * /*const*/ fastn = &code_state->state[n_state - 1]; + mp_exc_stack_t * /*const*/ exc_stack = (mp_exc_stack_t*)(code_state->state + n_state); // variables that are visible to the exception handler (declared volatile) volatile bool currently_in_except_block = MP_TAGPTR_TAG0(code_state->exc_sp); // 0 or 1, to detect nested exceptions @@ -1327,8 +1329,16 @@ unwind_loop: // But consider how to handle nested exceptions. // TODO need a better way of not adding traceback to constant objects (right now, just GeneratorExit_obj and MemoryError_obj) if (nlr.ret_val != &mp_const_GeneratorExit_obj && nlr.ret_val != &mp_const_MemoryError_obj) { - const byte *ip = code_state->code_info; + const byte *ip = code_state->fun_bc->bytecode; + mp_decode_uint(&ip); // skip n_state + mp_decode_uint(&ip); // skip n_exc_stack + ip++; // skip scope_params + ip++; // skip n_pos_args + ip++; // skip n_kwonly_args + ip++; // skip n_def_pos_args + size_t bc = code_state->ip - ip; size_t code_info_size = mp_decode_uint(&ip); + bc -= code_info_size; #if MICROPY_PERSISTENT_CODE qstr block_name = ip[0] | (ip[1] << 8); qstr source_file = ip[2] | (ip[3] << 8); @@ -1337,7 +1347,6 @@ unwind_loop: qstr block_name = mp_decode_uint(&ip); qstr source_file = mp_decode_uint(&ip); #endif - size_t bc = code_state->ip - code_state->code_info - code_info_size; size_t source_line = 1; size_t c; while ((c = *ip)) { @@ -1393,8 +1402,8 @@ unwind_loop: } else if (code_state->prev != NULL) { mp_globals_set(code_state->old_globals); code_state = code_state->prev; - fastn = &code_state->state[code_state->n_state - 1]; - exc_stack = (mp_exc_stack_t*)(code_state->state + code_state->n_state); + fastn = &code_state->state[n_state - 1]; + exc_stack = (mp_exc_stack_t*)(code_state->state + n_state); // variables that are visible to the exception handler (declared volatile) currently_in_except_block = MP_TAGPTR_TAG0(code_state->exc_sp); // 0 or 1, to detect nested exceptions exc_sp = MP_TAGPTR_PTR(code_state->exc_sp); // stack grows up, exc_sp points to top of stack From 5640e6dacd39d97adffce8490991c457f457b1cd Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Mar 2017 16:38:46 +1100 Subject: [PATCH 002/166] py: Provide mp_decode_uint_value to help optimise stack usage. This has a noticeable improvement on x86-64 and Thumb2 archs, where stack usage is reduced by 2 machine words in the VM. --- py/bc.c | 16 +++++++++++----- py/bc.h | 1 + py/objgenerator.c | 3 +-- py/vm.c | 3 +-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/py/bc.c b/py/bc.c index db5d0e6869..e7a1a333f7 100644 --- a/py/bc.c +++ b/py/bc.c @@ -54,6 +54,16 @@ mp_uint_t mp_decode_uint(const byte **ptr) { return unum; } +// This function is used to help reduce stack usage at the caller, for the case when +// the caller doesn't need to increase the ptr argument. If ptr is a local variable +// and the caller uses mp_decode_uint(&ptr) instead of this function, then the compiler +// must allocate a slot on the stack for ptr, and this slot cannot be reused for +// anything else in the function because the pointer may have been stored in a global +// and reused later in the function. +mp_uint_t mp_decode_uint_value(const byte *ptr) { + return mp_decode_uint(&ptr); +} + STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) { #if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE // generic message, used also for other argument issues @@ -246,11 +256,7 @@ continue2:; const byte *ip = code_state->ip; // jump over code info (source file and line-number mapping) - { - const byte *ip2 = ip; - size_t code_info_size = mp_decode_uint(&ip2); - ip += code_info_size; - } + ip += mp_decode_uint_value(ip); // bytecode prelude: initialise closed over variables size_t local_num; diff --git a/py/bc.h b/py/bc.h index 996b1a2f32..e8d4286125 100644 --- a/py/bc.h +++ b/py/bc.h @@ -91,6 +91,7 @@ typedef struct _mp_code_state_t { } mp_code_state_t; mp_uint_t mp_decode_uint(const byte **ptr); +mp_uint_t mp_decode_uint_value(const byte *ptr); mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); diff --git a/py/objgenerator.c b/py/objgenerator.c index 2baead2315..e59ff3a9ce 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -135,8 +135,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ break; case MP_VM_RETURN_EXCEPTION: { - const byte *bc = self->code_state.fun_bc->bytecode; - size_t n_state = mp_decode_uint(&bc); + size_t n_state = mp_decode_uint_value(self->code_state.fun_bc->bytecode); self->code_state.ip = 0; *ret_val = self->code_state.state[n_state - 1]; break; diff --git a/py/vm.c b/py/vm.c index 63a88a4fb5..4ec1ea9066 100644 --- a/py/vm.c +++ b/py/vm.c @@ -162,8 +162,7 @@ mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp run_code_state: ; #endif // Pointers which are constant for particular invocation of mp_execute_bytecode() - const byte *temp_bc = code_state->fun_bc->bytecode; - size_t n_state = mp_decode_uint(&temp_bc); + size_t n_state = mp_decode_uint_value(code_state->fun_bc->bytecode); mp_obj_t * /*const*/ fastn = &code_state->state[n_state - 1]; mp_exc_stack_t * /*const*/ exc_stack = (mp_exc_stack_t*)(code_state->state + n_state); From 9b80a1e3e93e51fdac8a288de58474e4b26efd46 Mon Sep 17 00:00:00 2001 From: Christopher Arndt Date: Sun, 5 Mar 2017 19:56:36 +0100 Subject: [PATCH 003/166] utime module documentation fixes and cleanup: * Fix mis-spelling of `ticks_add` in code examples. * Be consistent about parentheses after function names. * Be consistent about formatting of function, variable and constant names. * Be consistent about spaces and punctuation. * Fix some language errors (missing or wrong words, wrong word order). * Keep line length under 90 chars. Signed-off-by: Christopher Arndt --- docs/library/utime.rst | 107 +++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/docs/library/utime.rst b/docs/library/utime.rst index 109c3560cc..0b47a036e6 100644 --- a/docs/library/utime.rst +++ b/docs/library/utime.rst @@ -58,7 +58,7 @@ Functions .. only:: port_unix or port_pyboard or port_esp8266 .. function:: sleep(seconds) - + Sleep for the given number of seconds. Seconds can be a floating-point number to sleep for a fractional number of seconds. Note that other MicroPython ports may not accept floating-point argument, for compatibility with them use ``sleep_ms()`` @@ -67,32 +67,32 @@ Functions .. only:: port_wipy .. function:: sleep(seconds) - + Sleep for the given number of seconds. .. only:: port_unix or port_pyboard or port_wipy or port_esp8266 - .. function:: sleep_ms(ms) + .. function:: sleep_ms(ms) Delay for given number of milliseconds, should be positive or 0. - .. function:: sleep_us(us) + .. function:: sleep_us(us) - Delay for given number of microseconds, should be positive or 0 + Delay for given number of microseconds, should be positive or 0. - .. function:: ticks_ms() + .. function:: ticks_ms() - Returns an increasing millisecond counter with an arbitrary reference point, - that wraps around after some value. This value is not explicitly exposed, - but we will refer to it as `TICKS_MAX` to simplify discussion. Period of - the values is `TICKS_PERIOD = TICKS_MAX + 1`. `TICKS_PERIOD` is guaranteed - to be a power of two, but otherwise may differ from port to port. The same - period value is used for all of ticks_ms(), ticks_us(), ticks_cpu() functions - (for simplicity). Thus, these functions will return a value in range - [0 .. `TICKS_MAX`], inclusive, total `TICKS_PERIOD` values. Note that only - non-negative values are used. For the most part, you should treat values - returned by these functions as opaque. The only operations available for them - are ``ticks_diff()`` and ``ticks_add()`` functions described below. + Returns an increasing millisecond counter with an arbitrary reference point, that + wraps around after some value. This value is not explicitly exposed, but we will + refer to it as ``TICKS_MAX`` to simplify discussion. Period of the values is + ``TICKS_PERIOD = TICKS_MAX + 1``. ``TICKS_PERIOD`` is guaranteed to be a power of + two, but otherwise may differ from port to port. The same period value is used + for all of ``ticks_ms()``, ``ticks_us()``, ``ticks_cpu()`` functions (for + simplicity). Thus, these functions will return a value in range [``0`` .. + ``TICKS_MAX``], inclusive, total ``TICKS_PERIOD`` values. Note that only + non-negative values are used. For the most part, you should treat values returned + by these functions as opaque. The only operations available for them are + ``ticks_diff()`` and ``ticks_add()`` functions described below. Note: Performing standard mathematical operations (+, -) or relational operators (<, <=, >, >=) directly on these value will lead to invalid @@ -100,15 +100,15 @@ Functions as arguments to ``ticks_diff()`` or ``ticks_add()`` will also lead to invalid results from the latter functions. - .. function:: ticks_us() + .. function:: ticks_us() - Just like ``ticks_ms`` above, but in microseconds. + Just like ``ticks_ms()`` above, but in microseconds. -.. function:: ticks_cpu() +.. function:: ticks_cpu() - Similar to ``ticks_ms`` and ``ticks_us``, but with the highest possible resolution + Similar to ``ticks_ms()`` and ``ticks_us()``, but with the highest possible resolution in the system. This is usually CPU clocks, and that's why the function is named that - way. But it doesn't have to a CPU clock, some other timing source available in a + way. But it doesn't have to be a CPU clock, some other timing source available in a system (e.g. high-resolution timer) can be used instead. The exact timing unit (resolution) of this function is not specified on ``utime`` module level, but documentation for a specific port may provide more specific information. This @@ -118,13 +118,13 @@ Functions Availability: Not every port implements this function. -.. function:: ticks_add(ticks, delta) +.. function:: ticks_add(ticks, delta) Offset ticks value by a given number, which can be either positive or negative. Given a ``ticks`` value, this function allows to calculate ticks value ``delta`` ticks before or after it, following modular-arithmetic definition of tick values (see ``ticks_ms()`` above). ``ticks`` parameter must be a direct result of call - to ``tick_ms()``, ``ticks_us()``, ``ticks_cpu()`` functions (or from previous + to ``ticks_ms()``, ``ticks_us()``, or ``ticks_cpu()`` functions (or from previous call to ``ticks_add()``). However, ``delta`` can be an arbitrary integer number or numeric expression. ``ticks_add()`` is useful for calculating deadlines for events/tasks. (Note: you must use ``ticks_diff()`` function to work with @@ -133,35 +133,37 @@ Functions Examples:: # Find out what ticks value there was 100ms ago - print(tick_add(time.ticks_ms(), -100)) + print(ticks_add(time.ticks_ms(), -100)) # Calculate deadline for operation and test for it - deadline = tick_add(time.ticks_ms(), 200) + deadline = ticks_add(time.ticks_ms(), 200) while ticks_diff(deadline, time.ticks_ms()) > 0: do_a_little_of_something() # Find out TICKS_MAX used by this port - print(tick_add(0, -1)) + print(ticks_add(0, -1)) -.. function:: ticks_diff(ticks1, ticks2) +.. function:: ticks_diff(ticks1, ticks2) - Measure ticks difference between values returned from ticks_ms(), ticks_us(), or ticks_cpu() - functions. The argument order is the same as for subtraction operator, - ``tick_diff(ticks1, ticks2)`` has the same meaning as ``ticks1 - ticks2``. However, values returned by - ticks_ms(), etc. functions may wrap around, so directly using subtraction on them will - produce incorrect result. That is why ticks_diff() is needed, it implements modular - (or more specifically, ring) arithmetics to produce correct result even for wrap-around - values (as long as they not too distant inbetween, see below). The function returns - **signed** value in the range [`-TICKS_PERIOD/2` .. `TICKS_PERIOD/2-1`] (that's a typical - range definition for two's-complement signed binary integers). If the result is negative, - it means that `ticks1` occured earlier in time than `ticks2`. Otherwise, it means that - `ticks1` occured after `ticks2`. This holds `only` if `ticks1` and `ticks2` are apart from - each other for no more than `TICKS_PERIOD/2-1` ticks. If that does not hold, incorrect - result will be returned. Specifically, if 2 tick values are apart for `TICKS_PERIOD/2-1` - ticks, that value will be returned by the function. However, if `TICKS_PERIOD/2` of - real-time ticks has passed between them, the function will return `-TICKS_PERIOD/2` - instead, i.e. result value will wrap around to the negative range of possible values. + Measure ticks difference between values returned from ``ticks_ms()``, ``ticks_us()``, + or ``ticks_cpu()`` functions. The argument order is the same as for subtraction + operator, ``ticks_diff(ticks1, ticks2)`` has the same meaning as ``ticks1 - ticks2``. + However, values returned by ``ticks_ms()``, etc. functions may wrap around, so + directly using subtraction on them will produce incorrect result. That is why + ``ticks_diff()`` is needed, it implements modular (or more specifically, ring) + arithmetics to produce correct result even for wrap-around values (as long as they not + too distant inbetween, see below). The function returns **signed** value in the range + [``-TICKS_PERIOD/2`` .. ``TICKS_PERIOD/2-1``] (that's a typical range definition for + two's-complement signed binary integers). If the result is negative, it means that + ``ticks1`` occured earlier in time than ``ticks2``. Otherwise, it means that + ``ticks1`` occured after ``ticks2``. This holds ``only`` if ``ticks1`` and ``ticks2`` + are apart from each other for no more than ``TICKS_PERIOD/2-1`` ticks. If that does + not hold, incorrect result will be returned. Specifically, if two tick values are + apart for ``TICKS_PERIOD/2-1`` ticks, that value will be returned by the function. + However, if ``TICKS_PERIOD/2`` of real-time ticks has passed between them, the + function will return ``-TICKS_PERIOD/2`` instead, i.e. result value will wrap around + to the negative range of possible values. Informal rationale of the constraints above: Suppose you are locked in a room with no means to monitor passing of time except a standard 12-notch clock. Then if you look at @@ -200,20 +202,21 @@ Functions print("Oops, running late, tell task to run faster!") task.run(run_faster=true) - Note: Do not pass ``time()`` values to ``ticks_diff()``, and should use + Note: Do not pass ``time()`` values to ``ticks_diff()``, you should use normal mathematical operations on them. But note that ``time()`` may (and will) also overflow. This is known as https://en.wikipedia.org/wiki/Year_2038_problem . .. function:: time() - Returns the number of seconds, as an integer, since the Epoch, assuming that underlying - RTC is set and maintained as described above. If an RTC is not set, this function returns - number of seconds since a port-specific reference point in time (for embedded boards without - a battery-backed RTC, usually since power up or reset). If you want to develop portable - MicroPython application, you should not rely on this function to provide higher than second - precision. If you need higher precision, use ``ticks_ms()`` and ``ticks_us()`` functions, - if you need calendar time, ``localtime()`` without an argument is a better choice. + Returns the number of seconds, as an integer, since the Epoch, assuming that + underlying RTC is set and maintained as described above. If an RTC is not set, this + function returns number of seconds since a port-specific reference point in time (for + embedded boards without a battery-backed RTC, usually since power up or reset). If you + want to develop portable MicroPython application, you should not rely on this function + to provide higher than second precision. If you need higher precision, use + ``ticks_ms()`` and ``ticks_us()`` functions, if you need calendar time, + ``localtime()`` without an argument is a better choice. .. admonition:: Difference to CPython :class: attention From bf29fe2e138a30688cfb94ad3859f903f935ced1 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 15 Mar 2017 12:17:38 +0100 Subject: [PATCH 004/166] py/objstr: Use better msg in bad implicit str/bytes conversion exception Instead of always reporting some object cannot be implicitly be converted to a 'str', even when it is a 'bytes' object, adjust the logic so that when trying to convert str to bytes it is shown like that. This will still report bad implicit conversion from e.g. 'int to bytes' as 'int to str' but it will not result in the confusing 'can't convert 'str' object to str implicitly' anymore for calls like b'somestring'.count('a'). --- py/objstr.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index 2331004a5b..1e71617bda 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -2065,9 +2065,10 @@ STATIC void bad_implicit_conversion(mp_obj_t self_in) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_raise_TypeError("can't convert to str implicitly"); } else { + const qstr src_name = mp_obj_get_type(self_in)->name; nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, - "can't convert '%s' object to str implicitly", - mp_obj_get_type_str(self_in))); + "can't convert '%q' object to %q implicitly", + src_name, src_name == MP_QSTR_str ? MP_QSTR_bytes : MP_QSTR_str)); } } From 6e74d24f30bca3fb70876b1fb9a6b3a59850b83c Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 18:05:06 +1100 Subject: [PATCH 005/166] py: Add micropython.schedule() function and associated runtime code. --- lib/utils/interrupt_char.c | 5 ++ py/modmicropython.c | 14 +++++ py/mpconfig.h | 10 ++++ py/mpstate.h | 16 +++++ py/py.mk | 1 + py/runtime.c | 4 ++ py/runtime.h | 10 ++++ py/scheduler.c | 117 +++++++++++++++++++++++++++++++++++++ py/vm.c | 20 +++++++ 9 files changed, 197 insertions(+) create mode 100644 py/scheduler.c diff --git a/lib/utils/interrupt_char.c b/lib/utils/interrupt_char.c index ab4efd9113..344db88c72 100644 --- a/lib/utils/interrupt_char.c +++ b/lib/utils/interrupt_char.c @@ -46,4 +46,9 @@ void mp_keyboard_interrupt(void) { #else MP_STATE_VM(mp_pending_exception) = MP_STATE_PORT(mp_kbd_exception); #endif + #if MICROPY_ENABLE_SCHEDULER + if (MP_STATE_VM(sched_state) == MP_SCHED_IDLE) { + MP_STATE_VM(sched_state) = MP_SCHED_PENDING; + } + #endif } diff --git a/py/modmicropython.c b/py/modmicropython.c index 675d169cc4..a74e6aa3cb 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -29,6 +29,7 @@ #include "py/mpstate.h" #include "py/builtin.h" #include "py/stackctrl.h" +#include "py/runtime.h" #include "py/gc.h" // Various builtins specific to MicroPython runtime, @@ -128,6 +129,16 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_unlock_obj, mp_micropython_ STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_alloc_emergency_exception_buf_obj, mp_alloc_emergency_exception_buf); #endif +#if MICROPY_ENABLE_SCHEDULER +STATIC mp_obj_t mp_micropython_schedule(mp_obj_t function, mp_obj_t arg) { + if (!mp_sched_schedule(function, arg)) { + mp_raise_msg(&mp_type_RuntimeError, "schedule stack full"); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_micropython_schedule_obj, mp_micropython_schedule); +#endif + STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_micropython) }, { MP_ROM_QSTR(MP_QSTR_const), MP_ROM_PTR(&mp_identity_obj) }, @@ -151,6 +162,9 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_heap_lock), MP_ROM_PTR(&mp_micropython_heap_lock_obj) }, { MP_ROM_QSTR(MP_QSTR_heap_unlock), MP_ROM_PTR(&mp_micropython_heap_unlock_obj) }, #endif + #if MICROPY_ENABLE_SCHEDULER + { MP_ROM_QSTR(MP_QSTR_schedule), MP_ROM_PTR(&mp_micropython_schedule_obj) }, + #endif }; STATIC MP_DEFINE_CONST_DICT(mp_module_micropython_globals, mp_module_micropython_globals_table); diff --git a/py/mpconfig.h b/py/mpconfig.h index 7e3bd8bdbb..54cf0f3d3d 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -616,6 +616,16 @@ typedef double mp_float_t; #define MICROPY_USE_INTERNAL_PRINTF (1) #endif +// Support for internal scheduler +#ifndef MICROPY_ENABLE_SCHEDULER +#define MICROPY_ENABLE_SCHEDULER (0) +#endif + +// Maximum number of entries in the scheduler +#ifndef MICROPY_SCHEDULER_DEPTH +#define MICROPY_SCHEDULER_DEPTH (4) +#endif + // Support for generic VFS sub-system #ifndef MICROPY_VFS #define MICROPY_VFS (0) diff --git a/py/mpstate.h b/py/mpstate.h index 9e4f54ae5b..29f93870a8 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -50,6 +50,16 @@ typedef struct mp_dynamic_compiler_t { extern mp_dynamic_compiler_t mp_dynamic_compiler; #endif +// These are the values for sched_state +#define MP_SCHED_IDLE (1) +#define MP_SCHED_LOCKED (-1) +#define MP_SCHED_PENDING (0) // 0 so it's a quick check in the VM + +typedef struct _mp_sched_item_t { + mp_obj_t func; + mp_obj_t arg; +} mp_sched_item_t; + // This structure hold information about the memory allocation system. typedef struct _mp_state_mem_t { #if MICROPY_MEM_STATS @@ -129,6 +139,12 @@ typedef struct _mp_state_vm_t { // pending exception object (MP_OBJ_NULL if not pending) volatile mp_obj_t mp_pending_exception; + #if MICROPY_ENABLE_SCHEDULER + volatile int16_t sched_state; + uint16_t sched_sp; + mp_sched_item_t sched_stack[MICROPY_SCHEDULER_DEPTH]; + #endif + // current exception being handled, for sys.exc_info() #if MICROPY_PY_SYS_EXC_INFO mp_obj_base_t *cur_exception; diff --git a/py/py.mk b/py/py.mk index c4e2a79e66..37bb5d023e 100644 --- a/py/py.mk +++ b/py/py.mk @@ -143,6 +143,7 @@ PY_O_BASENAME = \ persistentcode.o \ runtime.o \ runtime_utils.o \ + scheduler.o \ nativeglue.o \ stackctrl.o \ argcheck.o \ diff --git a/py/runtime.c b/py/runtime.c index 1f69290ba7..19b55f99ac 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -63,6 +63,10 @@ void mp_init(void) { // no pending exceptions to start with MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; + #if MICROPY_ENABLE_SCHEDULER + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; + MP_STATE_VM(sched_sp) = 0; + #endif #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF mp_init_emergency_exception_buf(); diff --git a/py/runtime.h b/py/runtime.h index 954833b67a..d4b1fa79a6 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -64,6 +64,16 @@ extern const qstr mp_binary_op_method_name[]; void mp_init(void); void mp_deinit(void); +void mp_handle_pending(void); +void mp_handle_pending_tail(mp_uint_t atomic_state); + +#if MICROPY_ENABLE_SCHEDULER +void mp_sched_lock(void); +void mp_sched_unlock(void); +static inline unsigned int mp_sched_num_pending(void) { return MP_STATE_VM(sched_sp); } +bool mp_sched_schedule(mp_obj_t function, mp_obj_t arg); +#endif + // extra printing method specifically for mp_obj_t's which are integral type int mp_print_mp_int(const mp_print_t *print, mp_obj_t x, int base, int base_char, int flags, char fill, int width, int prec); diff --git a/py/scheduler.c b/py/scheduler.c new file mode 100644 index 0000000000..30851a4d2b --- /dev/null +++ b/py/scheduler.c @@ -0,0 +1,117 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/runtime.h" + +#if MICROPY_ENABLE_SCHEDULER + +// A variant of this is inlined in the VM at the pending exception check +void mp_handle_pending(void) { + if (MP_STATE_VM(sched_state) == MP_SCHED_PENDING) { + mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + mp_obj_t obj = MP_STATE_VM(mp_pending_exception); + if (obj != MP_OBJ_NULL) { + MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; + if (!mp_sched_num_pending()) { + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; + } + MICROPY_END_ATOMIC_SECTION(atomic_state); + nlr_raise(obj); + } + mp_handle_pending_tail(atomic_state); + } +} + +// This function should only be called be mp_sched_handle_pending, +// or by the VM's inlined version of that function. +void mp_handle_pending_tail(mp_uint_t atomic_state) { + MP_STATE_VM(sched_state) = MP_SCHED_LOCKED; + if (MP_STATE_VM(sched_sp) > 0) { + mp_sched_item_t item = MP_STATE_VM(sched_stack)[--MP_STATE_VM(sched_sp)]; + MICROPY_END_ATOMIC_SECTION(atomic_state); + mp_call_function_1_protected(item.func, item.arg); + } else { + MICROPY_END_ATOMIC_SECTION(atomic_state); + } + mp_sched_unlock(); +} + +void mp_sched_lock(void) { + mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + if (MP_STATE_VM(sched_state) < 0) { + --MP_STATE_VM(sched_state); + } else { + MP_STATE_VM(sched_state) = MP_SCHED_LOCKED; + } + MICROPY_END_ATOMIC_SECTION(atomic_state); +} + +void mp_sched_unlock(void) { + mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + if (++MP_STATE_VM(sched_state) == 0) { + // vm became unlocked + if (MP_STATE_VM(mp_pending_exception) != MP_OBJ_NULL || mp_sched_num_pending()) { + MP_STATE_VM(sched_state) = MP_SCHED_PENDING; + } else { + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; + } + } + MICROPY_END_ATOMIC_SECTION(atomic_state); +} + +bool mp_sched_schedule(mp_obj_t function, mp_obj_t arg) { + mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + bool ret; + if (MP_STATE_VM(sched_sp) < MICROPY_SCHEDULER_DEPTH) { + if (MP_STATE_VM(sched_state) == MP_SCHED_IDLE) { + MP_STATE_VM(sched_state) = MP_SCHED_PENDING; + } + MP_STATE_VM(sched_stack)[MP_STATE_VM(sched_sp)].func = function; + MP_STATE_VM(sched_stack)[MP_STATE_VM(sched_sp)].arg = arg; + ++MP_STATE_VM(sched_sp); + ret = true; + } else { + // schedule stack is full + ret = false; + } + MICROPY_END_ATOMIC_SECTION(atomic_state); + return ret; +} + +#else // MICROPY_ENABLE_SCHEDULER + +// A variant of this is inlined in the VM at the pending exception check +void mp_handle_pending(void) { + if (MP_STATE_VM(mp_pending_exception) != MP_OBJ_NULL) { + mp_obj_t obj = MP_STATE_VM(mp_pending_exception); + MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; + nlr_raise(obj); + } +} + +#endif // MICROPY_ENABLE_SCHEDULER diff --git a/py/vm.c b/py/vm.c index 4ec1ea9066..98eababc04 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1267,12 +1267,32 @@ yield: pending_exception_check: MICROPY_VM_HOOK_LOOP + + #if MICROPY_ENABLE_SCHEDULER + // This is an inlined variant of mp_handle_pending + if (MP_STATE_VM(sched_state) == MP_SCHED_PENDING) { + MARK_EXC_IP_SELECTIVE(); + mp_uint_t atomic_state = MICROPY_BEGIN_ATOMIC_SECTION(); + mp_obj_t obj = MP_STATE_VM(mp_pending_exception); + if (obj != MP_OBJ_NULL) { + MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; + if (!mp_sched_num_pending()) { + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; + } + MICROPY_END_ATOMIC_SECTION(atomic_state); + RAISE(obj); + } + mp_handle_pending_tail(atomic_state); + } + #else + // This is an inlined variant of mp_handle_pending if (MP_STATE_VM(mp_pending_exception) != MP_OBJ_NULL) { MARK_EXC_IP_SELECTIVE(); mp_obj_t obj = MP_STATE_VM(mp_pending_exception); MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; RAISE(obj); } + #endif #if MICROPY_PY_THREAD_GIL #if MICROPY_PY_THREAD_GIL_VM_DIVISOR From c138b21cebfa1a32e9911c1424aa1016c679463b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 23:05:38 +1100 Subject: [PATCH 006/166] unix: Use mp_handle_pending() in time.sleep(). --- unix/modtime.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/unix/modtime.c b/unix/modtime.c index 85d1f55327..080d321ee4 100644 --- a/unix/modtime.c +++ b/unix/modtime.c @@ -108,9 +108,7 @@ STATIC mp_obj_t mod_time_sleep(mp_obj_t arg) { if (res != -1 || errno != EINTR) { break; } - if (MP_STATE_VM(mp_pending_exception) != MP_OBJ_NULL) { - return mp_const_none; - } + mp_handle_pending(); //printf("select: EINTR: %ld:%ld\n", tv.tv_sec, tv.tv_usec); #else break; From a5159edc2090a5670c33a829d7e54ab2ba8635c4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 23:04:53 +1100 Subject: [PATCH 007/166] stmhal: Enable micropython.schedule(). ExtInt, Timer and CAN IRQ callbacks are made to work with the scheduler. They are still hard IRQs by default, but one can now call micropython.schedule within the hard IRQ to schedule a soft callback. --- stmhal/can.c | 2 ++ stmhal/extint.c | 3 ++- stmhal/mpconfigport.h | 11 ++++++++++- stmhal/systick.c | 2 ++ stmhal/timer.c | 2 ++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/stmhal/can.c b/stmhal/can.c index e293b87425..9b8f2a0715 100644 --- a/stmhal/can.c +++ b/stmhal/can.c @@ -865,6 +865,7 @@ void can_rx_irq_handler(uint can_id, uint fifo_id) { } if (callback != mp_const_none) { + mp_sched_lock(); gc_lock(); nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { @@ -877,6 +878,7 @@ void can_rx_irq_handler(uint can_id, uint fifo_id) { mp_obj_print_exception(&mp_plat_print, (mp_obj_t)nlr.ret_val); } gc_unlock(); + mp_sched_unlock(); } } diff --git a/stmhal/extint.c b/stmhal/extint.c index dacf8dd568..59b00cb73c 100644 --- a/stmhal/extint.c +++ b/stmhal/extint.c @@ -28,7 +28,6 @@ #include #include -#include "py/nlr.h" #include "py/runtime.h" #include "py/gc.h" #include "py/mphal.h" @@ -412,6 +411,7 @@ void Handle_EXTI_Irq(uint32_t line) { if (line < EXTI_NUM_VECTORS) { mp_obj_t *cb = &MP_STATE_PORT(pyb_extint_callback)[line]; if (*cb != mp_const_none) { + mp_sched_lock(); // When executing code within a handler we must lock the GC to prevent // any memory allocations. We must also catch any exceptions. gc_lock(); @@ -427,6 +427,7 @@ void Handle_EXTI_Irq(uint32_t line) { mp_obj_print_exception(&mp_plat_print, (mp_obj_t)nlr.ret_val); } gc_unlock(); + mp_sched_unlock(); } } } diff --git a/stmhal/mpconfigport.h b/stmhal/mpconfigport.h index 38a5ac5da9..339360baef 100644 --- a/stmhal/mpconfigport.h +++ b/stmhal/mpconfigport.h @@ -69,6 +69,8 @@ #define MICROPY_MODULE_WEAK_LINKS (1) #define MICROPY_CAN_OVERRIDE_BUILTINS (1) #define MICROPY_USE_INTERNAL_ERRNO (1) +#define MICROPY_ENABLE_SCHEDULER (1) +#define MICROPY_SCHEDULER_DEPTH (8) #define MICROPY_VFS (1) #define MICROPY_VFS_FAT (1) @@ -303,6 +305,8 @@ static inline mp_uint_t disable_irq(void) { #if MICROPY_PY_THREAD #define MICROPY_EVENT_POLL_HOOK \ do { \ + extern void mp_handle_pending(void); \ + mp_handle_pending(); \ if (pyb_thread_enabled) { \ MP_THREAD_GIL_EXIT(); \ pyb_thread_yield(); \ @@ -312,7 +316,12 @@ static inline mp_uint_t disable_irq(void) { } \ } while (0); #else -#define MICROPY_EVENT_POLL_HOOK __WFI(); +#define MICROPY_EVENT_POLL_HOOK \ + do { \ + extern void mp_handle_pending(void); \ + mp_handle_pending(); \ + __WFI(); \ + } while (0); #endif // There is no classical C heap in bare-metal ports, only Python diff --git a/stmhal/systick.c b/stmhal/systick.c index eb11de9b74..71e3d34889 100644 --- a/stmhal/systick.c +++ b/stmhal/systick.c @@ -24,6 +24,7 @@ * THE SOFTWARE. */ +#include "py/runtime.h" #include "py/mphal.h" #include "irq.h" #include "systick.h" @@ -58,6 +59,7 @@ void mp_hal_delay_ms(mp_uint_t Delay) { // Wraparound of tick is taken care of by 2's complement arithmetic. while (uwTick - start < Delay) { // Enter sleep mode, waiting for (at least) the SysTick interrupt. + mp_handle_pending(); #if MICROPY_PY_THREAD if (pyb_thread_enabled) { pyb_thread_yield(); diff --git a/stmhal/timer.c b/stmhal/timer.c index 494045c28f..7f0a70c5e8 100644 --- a/stmhal/timer.c +++ b/stmhal/timer.c @@ -1363,6 +1363,7 @@ STATIC void timer_handle_irq_channel(pyb_timer_obj_t *tim, uint8_t channel, mp_o // execute callback if it's set if (callback != mp_const_none) { + mp_sched_lock(); // When executing code within a handler we must lock the GC to prevent // any memory allocations. We must also catch any exceptions. gc_lock(); @@ -1382,6 +1383,7 @@ STATIC void timer_handle_irq_channel(pyb_timer_obj_t *tim, uint8_t channel, mp_o mp_obj_print_exception(&mp_plat_print, (mp_obj_t)nlr.ret_val); } gc_unlock(); + mp_sched_unlock(); } } } From 1b7d67266df7dd0853698726e98d63b4aeebe205 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 17:45:36 +1100 Subject: [PATCH 008/166] esp8266: Enable micropython.schedule() with locking in pin callback. --- esp8266/esp8266_common.ld | 1 + esp8266/esp_mphal.c | 7 ++----- esp8266/machine_pin.c | 2 ++ esp8266/mpconfigport.h | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/esp8266/esp8266_common.ld b/esp8266/esp8266_common.ld index f721c28b03..1da835681a 100644 --- a/esp8266/esp8266_common.ld +++ b/esp8266/esp8266_common.ld @@ -100,6 +100,7 @@ SECTIONS *py/qstr.o*(.literal* .text*) *py/repl.o*(.literal* .text*) *py/runtime.o*(.literal* .text*) + *py/scheduler.o*(.literal* .text*) *py/scope.o*(.literal* .text*) *py/sequence.o*(.literal* .text*) *py/showbc.o*(.literal* .text*) diff --git a/esp8266/esp_mphal.c b/esp8266/esp_mphal.c index f5e284fde6..7ecc7776aa 100644 --- a/esp8266/esp_mphal.c +++ b/esp8266/esp_mphal.c @@ -33,6 +33,7 @@ #include "ets_alt_task.h" #include "py/obj.h" #include "py/mpstate.h" +#include "py/runtime.h" #include "extmod/misc.h" #include "lib/utils/pyexec.h" @@ -130,11 +131,7 @@ void mp_hal_delay_ms(uint32_t delay) { void ets_event_poll(void) { ets_loop_iter(); - if (MP_STATE_VM(mp_pending_exception) != NULL) { - mp_obj_t obj = MP_STATE_VM(mp_pending_exception); - MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; - nlr_raise(obj); - } + mp_handle_pending(); } void __assert_func(const char *file, int line, const char *func, const char *expr) { diff --git a/esp8266/machine_pin.c b/esp8266/machine_pin.c index a1e94e898e..9ea5197bc8 100644 --- a/esp8266/machine_pin.c +++ b/esp8266/machine_pin.c @@ -100,6 +100,7 @@ void pin_init0(void) { } void pin_intr_handler(uint32_t status) { + mp_sched_lock(); gc_lock(); status &= 0xffff; for (int p = 0; status; ++p, status >>= 1) { @@ -111,6 +112,7 @@ void pin_intr_handler(uint32_t status) { } } gc_unlock(); + mp_sched_unlock(); } pyb_pin_obj_t *mp_obj_get_pin_obj(mp_obj_t pin_in) { diff --git a/esp8266/mpconfigport.h b/esp8266/mpconfigport.h index cf4cbecd40..07bd48f679 100644 --- a/esp8266/mpconfigport.h +++ b/esp8266/mpconfigport.h @@ -28,6 +28,7 @@ #define MICROPY_MODULE_WEAK_LINKS (1) #define MICROPY_CAN_OVERRIDE_BUILTINS (1) #define MICROPY_USE_INTERNAL_ERRNO (1) +#define MICROPY_ENABLE_SCHEDULER (1) #define MICROPY_PY_ALL_SPECIAL_METHODS (1) #define MICROPY_PY_BUILTINS_COMPLEX (0) #define MICROPY_PY_BUILTINS_STR_UNICODE (1) From 31ea15855712ee6a69b85b17aba0607118398e4f Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Feb 2017 17:59:18 +1100 Subject: [PATCH 009/166] esp8266: Change machine.Timer callback to soft callback. --- esp8266/modmachine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp8266/modmachine.c b/esp8266/modmachine.c index 222ed004d8..c26c633967 100644 --- a/esp8266/modmachine.c +++ b/esp8266/modmachine.c @@ -157,7 +157,7 @@ STATIC mp_obj_t esp_timer_make_new(const mp_obj_type_t *type, size_t n_args, siz STATIC void esp_timer_cb(void *arg) { esp_timer_obj_t *self = arg; - mp_call_function_1_protected(self->callback, self); + mp_sched_schedule(self->callback, self); } STATIC mp_obj_t esp_timer_init_helper(esp_timer_obj_t *self, mp_uint_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { From 2507c83b0ebb9f3876b076eeb4bca992832e2aa2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 17:45:53 +1100 Subject: [PATCH 010/166] esp8266/machine_pin: Add "hard" parameter to pin.irq, soft by default. --- esp8266/machine_pin.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/esp8266/machine_pin.c b/esp8266/machine_pin.c index 9ea5197bc8..0f5057d220 100644 --- a/esp8266/machine_pin.c +++ b/esp8266/machine_pin.c @@ -87,11 +87,15 @@ STATIC uint8_t pin_mode[16 + 1]; // forward declaration STATIC const pin_irq_obj_t pin_irq_obj[16]; +// whether the irq is hard or soft +STATIC bool pin_irq_is_hard[16]; + void pin_init0(void) { ETS_GPIO_INTR_DISABLE(); ETS_GPIO_INTR_ATTACH(pin_intr_handler_iram, NULL); // disable all interrupts memset(&MP_STATE_PORT(pin_irq_handler)[0], 0, 16 * sizeof(mp_obj_t)); + memset(pin_irq_is_hard, 0, sizeof(pin_irq_obj)); for (int p = 0; p < 16; ++p) { GPIO_REG_WRITE(GPIO_STATUS_W1TC_ADDRESS, 1 << p); SET_TRIGGER(p, 0); @@ -107,7 +111,11 @@ void pin_intr_handler(uint32_t status) { if (status & 1) { mp_obj_t handler = MP_STATE_PORT(pin_irq_handler)[p]; if (handler != MP_OBJ_NULL) { - mp_call_function_1_protected(handler, MP_OBJ_FROM_PTR(&pyb_pin_obj[p])); + if (pin_irq_is_hard[p]) { + mp_call_function_1_protected(handler, MP_OBJ_FROM_PTR(&pyb_pin_obj[p])); + } else { + mp_sched_schedule(handler, MP_OBJ_FROM_PTR(&pyb_pin_obj[p])); + } } } } @@ -346,10 +354,11 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(pyb_pin_high_obj, pyb_pin_high); // pin.irq(*, trigger, handler=None) STATIC mp_obj_t pyb_pin_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { - enum { ARG_trigger, ARG_handler }; + enum { ARG_trigger, ARG_handler, ARG_hard }; static const mp_arg_t allowed_args[] = { { MP_QSTR_trigger, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, { MP_QSTR_handler, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = mp_const_none} }, + { MP_QSTR_hard, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} }, }; pyb_pin_obj_t *self = MP_OBJ_TO_PTR(pos_args[0]); mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; @@ -367,6 +376,7 @@ STATIC mp_obj_t pyb_pin_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_t *k } ETS_GPIO_INTR_DISABLE(); MP_STATE_PORT(pin_irq_handler)[self->phys_port] = handler; + pin_irq_is_hard[self->phys_port] = args[ARG_hard].u_bool; SET_TRIGGER(self->phys_port, args[ARG_trigger].u_int); GPIO_REG_WRITE(GPIO_STATUS_W1TC_ADDRESS, 1 << self->phys_port); ETS_GPIO_INTR_ENABLE(); From 9ee464185064b287bc1b0ff42e1b81214d808e7a Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Mar 2017 17:15:34 +1100 Subject: [PATCH 011/166] esp8266/machine_pin: Make pin.irq arguments positional. All arguments to pin.irq are converted from keyword-only to positional, and can still be specified by keyword so it's a backwards compatible change. The default value for the "trigger" arg is changed from 0 (no trigger) to rising+falling edge. --- esp8266/machine_pin.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/esp8266/machine_pin.c b/esp8266/machine_pin.c index 0f5057d220..f2e334b1ae 100644 --- a/esp8266/machine_pin.c +++ b/esp8266/machine_pin.c @@ -352,13 +352,13 @@ STATIC mp_obj_t pyb_pin_high(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(pyb_pin_high_obj, pyb_pin_high); -// pin.irq(*, trigger, handler=None) +// pin.irq(handler=None, trigger=IRQ_FALLING|IRQ_RISING, hard=False) STATIC mp_obj_t pyb_pin_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { - enum { ARG_trigger, ARG_handler, ARG_hard }; + enum { ARG_handler, ARG_trigger, ARG_hard }; static const mp_arg_t allowed_args[] = { - { MP_QSTR_trigger, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, - { MP_QSTR_handler, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = mp_const_none} }, - { MP_QSTR_hard, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} }, + { MP_QSTR_handler, MP_ARG_OBJ, {.u_obj = mp_const_none} }, + { MP_QSTR_trigger, MP_ARG_INT, {.u_int = GPIO_PIN_INTR_POSEDGE | GPIO_PIN_INTR_NEGEDGE} }, + { MP_QSTR_hard, MP_ARG_BOOL, {.u_bool = false} }, }; pyb_pin_obj_t *self = MP_OBJ_TO_PTR(pos_args[0]); mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; @@ -368,16 +368,18 @@ STATIC mp_obj_t pyb_pin_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_t *k nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "pin does not have IRQ capabilities")); } - if (args[ARG_trigger].u_int != 0) { + if (n_args > 1) { // configure irq mp_obj_t handler = args[ARG_handler].u_obj; + uint32_t trigger = args[ARG_trigger].u_int; if (handler == mp_const_none) { handler = MP_OBJ_NULL; + trigger = 0; } ETS_GPIO_INTR_DISABLE(); MP_STATE_PORT(pin_irq_handler)[self->phys_port] = handler; pin_irq_is_hard[self->phys_port] = args[ARG_hard].u_bool; - SET_TRIGGER(self->phys_port, args[ARG_trigger].u_int); + SET_TRIGGER(self->phys_port, trigger); GPIO_REG_WRITE(GPIO_STATUS_W1TC_ADDRESS, 1 << self->phys_port); ETS_GPIO_INTR_ENABLE(); } From c772817deea6f3e5fe63483db77a98b65974d833 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Mar 2017 18:05:00 +1100 Subject: [PATCH 012/166] tests/micropython: Add tests for micropython.schedule(). --- tests/micropython/schedule.py | 61 +++++++++++++++++++++++++++++++ tests/micropython/schedule.py.exp | 4 ++ tests/run-tests | 1 + 3 files changed, 66 insertions(+) create mode 100644 tests/micropython/schedule.py create mode 100644 tests/micropython/schedule.py.exp diff --git a/tests/micropython/schedule.py b/tests/micropython/schedule.py new file mode 100644 index 0000000000..3d584eea4f --- /dev/null +++ b/tests/micropython/schedule.py @@ -0,0 +1,61 @@ +# test micropython.schedule() function + +import micropython + +try: + micropython.schedule +except AttributeError: + print('SKIP') + import sys + sys.exit() + +# Basic test of scheduling a function. + +def callback(arg): + global done + print(arg) + done = True + +done = False +micropython.schedule(callback, 1) +while not done: + pass + +# Test that callbacks can be scheduled from within a callback, but +# that they don't execute until the outer callback is finished. + +def callback_inner(arg): + global done + print('inner') + done += 1 + +def callback_outer(arg): + global done + micropython.schedule(callback_inner, 0) + # need a loop so that the VM can check for pending events + for i in range(2): + pass + print('outer') + done += 1 + +done = 0 +micropython.schedule(callback_outer, 0) +while done != 2: + pass + +# Test that scheduling too many callbacks leads to an exception. To do this we +# must schedule from within a callback to guarantee that the scheduler is locked. + +def callback(arg): + global done + try: + for i in range(100): + micropython.schedule(lambda x:x, None) + except RuntimeError: + print('RuntimeError') + done = True + +done = False +micropython.schedule(callback, None) +while not done: + pass diff --git a/tests/micropython/schedule.py.exp b/tests/micropython/schedule.py.exp new file mode 100644 index 0000000000..c4a3e1227e --- /dev/null +++ b/tests/micropython/schedule.py.exp @@ -0,0 +1,4 @@ +1 +outer +inner +RuntimeError diff --git a/tests/run-tests b/tests/run-tests index b1bb7dd846..e8dc2ba580 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -319,6 +319,7 @@ def run_tests(pyb, tests, args): skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info skip_tests.add('micropython/heapalloc_iter.py') # requires generators + skip_tests.add('micropython/schedule.py') # native code doesn't check pending events for test_file in tests: test_file = test_file.replace('\\', '/') From 74faf4c5fc2245e33a0deed840716a36249eb683 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Mar 2017 18:05:33 +1100 Subject: [PATCH 013/166] unix/coverage: Enable scheduler and add tests for it. --- tests/unix/extra_coverage.py.exp | 11 +++++++++++ unix/coverage.c | 29 +++++++++++++++++++++++++++++ unix/mpconfigport_coverage.h | 1 + 3 files changed, 41 insertions(+) diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 32117aba44..4169938870 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -46,6 +46,17 @@ Warning: test # binary 122 456 +# scheduler +sched(0)=1 +sched(1)=1 +sched(2)=1 +sched(3)=1 +sched(4)=0 +unlocked +3 +2 +1 +0 0123456789 b'0123456789' 7300 7300 diff --git a/unix/coverage.c b/unix/coverage.c index ca236c4303..09959525a0 100644 --- a/unix/coverage.c +++ b/unix/coverage.c @@ -292,6 +292,35 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "%.0lf\n", dar[0]); } + // scheduler + { + mp_printf(&mp_plat_print, "# scheduler\n"); + + // lock scheduler + mp_sched_lock(); + + // schedule multiple callbacks; last one should fail + for (int i = 0; i < 5; ++i) { + mp_printf(&mp_plat_print, "sched(%d)=%d\n", i, mp_sched_schedule(MP_OBJ_FROM_PTR(&mp_builtin_print_obj), MP_OBJ_NEW_SMALL_INT(i))); + } + + // test nested locking/unlocking + mp_sched_lock(); + mp_sched_unlock(); + + // shouldn't do anything while scheduler is locked + mp_handle_pending(); + + // unlock scheduler + mp_sched_unlock(); + mp_printf(&mp_plat_print, "unlocked\n"); + + // drain pending callbacks + while (mp_sched_num_pending()) { + mp_handle_pending(); + } + } + mp_obj_streamtest_t *s = m_new_obj(mp_obj_streamtest_t); s->base.type = &mp_type_stest_fileio; s->buf = NULL; diff --git a/unix/mpconfigport_coverage.h b/unix/mpconfigport_coverage.h index 9df8d0fca8..387e182db2 100644 --- a/unix/mpconfigport_coverage.h +++ b/unix/mpconfigport_coverage.h @@ -32,6 +32,7 @@ #include +#define MICROPY_ENABLE_SCHEDULER (1) #define MICROPY_PY_DELATTR_SETATTR (1) #define MICROPY_PY_BUILTINS_HELP (1) #define MICROPY_PY_BUILTINS_HELP_MODULES (1) From fb981107eb6eef3af2b7a6b1c8da53e95c316e86 Mon Sep 17 00:00:00 2001 From: transistortim Date: Sun, 19 Mar 2017 00:09:58 +0100 Subject: [PATCH 014/166] docs/library/machine.I2C: Fix scan() doc to match implementation. Since eaef6b5324fa2ff425802d4abeea45aa945bfc14 writes are used instead of reads. --- docs/library/machine.I2C.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/library/machine.I2C.rst b/docs/library/machine.I2C.rst index cdeb246ebb..45944709ef 100644 --- a/docs/library/machine.I2C.rst +++ b/docs/library/machine.I2C.rst @@ -100,7 +100,7 @@ General Methods Scan all I2C addresses between 0x08 and 0x77 inclusive and return a list of those that respond. A device responds if it pulls the SDA line low after - its address (including a read bit) is sent on the bus. + its address (including a write bit) is sent on the bus. Note: on WiPy the I2C object must be in master mode for this method to be valid. From 231cfc84a7cae31f93208c334fc33b08278040eb Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Mon, 6 Feb 2017 10:59:44 +0000 Subject: [PATCH 015/166] extmod/modframebuf: Add support for monochrome horizontal format. MHLSB and MHMSB formats are added to the framebuf module, which have 8 adjacent horizontal pixels represented in a single byte. --- extmod/modframebuf.c | 48 ++++++++++-- tests/extmod/framebuf1.py | 143 ++++++++++++++++++---------------- tests/extmod/framebuf1.py.exp | 46 +++++++++++ 3 files changed, 165 insertions(+), 72 deletions(-) diff --git a/extmod/modframebuf.c b/extmod/modframebuf.c index 792a3a7fa2..33985dd00a 100644 --- a/extmod/modframebuf.c +++ b/extmod/modframebuf.c @@ -53,6 +53,41 @@ typedef struct _mp_framebuf_p_t { fill_rect_t fill_rect; } mp_framebuf_p_t; +// constants for formats +#define FRAMEBUF_MVLSB (0) +#define FRAMEBUF_RGB565 (1) +#define FRAMEBUF_GS4_HMSB (2) +#define FRAMEBUF_MHLSB (3) +#define FRAMEBUF_MHMSB (4) + +// Functions for MHLSB and MHMSB + +STATIC void mono_horiz_setpixel(const mp_obj_framebuf_t *fb, int x, int y, uint32_t color) { + size_t index = (x + y * fb->stride) >> 3; + int offset = fb->format == FRAMEBUF_MHMSB ? x & 0x07 : 7 - (x & 0x07); + ((uint8_t*)fb->buf)[index] = (((uint8_t*)fb->buf)[index] & ~(0x01 << offset)) | ((color != 0) << offset); +} + +STATIC uint32_t mono_horiz_getpixel(const mp_obj_framebuf_t *fb, int x, int y) { + size_t index = (x + y * fb->stride) >> 3; + int offset = fb->format == FRAMEBUF_MHMSB ? x & 0x07 : 7 - (x & 0x07); + return (((uint8_t*)fb->buf)[index] >> (offset)) & 0x01; +} + +STATIC void mono_horiz_fill_rect(const mp_obj_framebuf_t *fb, int x, int y, int w, int h, uint32_t col) { + int reverse = fb->format == FRAMEBUF_MHMSB; + int advance = fb->stride >> 3; + while (w--) { + uint8_t *b = &((uint8_t*)fb->buf)[(x >> 3) + y * advance]; + int offset = reverse ? x & 7 : 7 - (x & 7); + for (int hh = h; hh; --hh) { + *b = (*b & ~(0x01 << offset)) | ((col != 0) << offset); + b += advance; + } + ++x; + } +} + // Functions for MVLSB format STATIC void mvlsb_setpixel(const mp_obj_framebuf_t *fb, int x, int y, uint32_t color) { @@ -148,15 +183,12 @@ STATIC void gs4_hmsb_fill_rect(const mp_obj_framebuf_t *fb, int x, int y, int w, } } -// constants for formats -#define FRAMEBUF_MVLSB (0) -#define FRAMEBUF_RGB565 (1) -#define FRAMEBUF_GS4_HMSB (2) - STATIC mp_framebuf_p_t formats[] = { [FRAMEBUF_MVLSB] = {mvlsb_setpixel, mvlsb_getpixel, mvlsb_fill_rect}, [FRAMEBUF_RGB565] = {rgb565_setpixel, rgb565_getpixel, rgb565_fill_rect}, [FRAMEBUF_GS4_HMSB] = {gs4_hmsb_setpixel, gs4_hmsb_getpixel, gs4_hmsb_fill_rect}, + [FRAMEBUF_MHLSB] = {mono_horiz_setpixel, mono_horiz_getpixel, mono_horiz_fill_rect}, + [FRAMEBUF_MHMSB] = {mono_horiz_setpixel, mono_horiz_getpixel, mono_horiz_fill_rect}, }; static inline void setpixel(const mp_obj_framebuf_t *fb, int x, int y, uint32_t color) { @@ -207,6 +239,10 @@ STATIC mp_obj_t framebuf_make_new(const mp_obj_type_t *type, size_t n_args, size case FRAMEBUF_RGB565: case FRAMEBUF_GS4_HMSB: break; + case FRAMEBUF_MHLSB: + case FRAMEBUF_MHMSB: + o->stride = (o->stride + 7) & ~7; + break; default: nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, "invalid format")); @@ -545,6 +581,8 @@ STATIC const mp_rom_map_elem_t framebuf_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_MVLSB), MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MVLSB) }, { MP_ROM_QSTR(MP_QSTR_RGB565), MP_OBJ_NEW_SMALL_INT(FRAMEBUF_RGB565) }, { MP_ROM_QSTR(MP_QSTR_GS4_HMSB), MP_OBJ_NEW_SMALL_INT(FRAMEBUF_GS4_HMSB) }, + { MP_ROM_QSTR(MP_QSTR_MHLSB), MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MHLSB) }, + { MP_ROM_QSTR(MP_QSTR_MHMSB), MP_OBJ_NEW_SMALL_INT(FRAMEBUF_MHMSB) }, }; STATIC MP_DEFINE_CONST_DICT(framebuf_module_globals, framebuf_module_globals_table); diff --git a/tests/extmod/framebuf1.py b/tests/extmod/framebuf1.py index c204e63aa3..0a8e1ae550 100644 --- a/tests/extmod/framebuf1.py +++ b/tests/extmod/framebuf1.py @@ -7,87 +7,96 @@ except ImportError: w = 5 h = 16 -buf = bytearray(w * h // 8) -fbuf = framebuf.FrameBuffer(buf, w, h, framebuf.MVLSB) +size = w * h // 8 +buf = bytearray(size) +maps = {framebuf.MVLSB : 'MVLSB', + framebuf.MHLSB : 'MHLSB', + framebuf.MHMSB : 'MHMSB'} -# access as buffer -print(memoryview(fbuf)[0]) +for mapping in maps.keys(): + for x in range(size): + buf[x] = 0 + fbuf = framebuf.FrameBuffer(buf, w, h, mapping) + print(maps[mapping]) + # access as buffer + print(memoryview(fbuf)[0]) -# fill -fbuf.fill(1) -print(buf) -fbuf.fill(0) -print(buf) + # fill + fbuf.fill(1) + print(buf) + fbuf.fill(0) + print(buf) -# put pixel -fbuf.pixel(0, 0, 1) -fbuf.pixel(4, 0, 1) -fbuf.pixel(0, 15, 1) -fbuf.pixel(4, 15, 1) -print(buf) + # put pixel + fbuf.pixel(0, 0, 1) + fbuf.pixel(4, 0, 1) + fbuf.pixel(0, 15, 1) + fbuf.pixel(4, 15, 1) + print(buf) -# clear pixel -fbuf.pixel(4, 15, 0) -print(buf) + # clear pixel + fbuf.pixel(4, 15, 0) + print(buf) -# get pixel -print(fbuf.pixel(0, 0), fbuf.pixel(1, 1)) + # get pixel + print(fbuf.pixel(0, 0), fbuf.pixel(1, 1)) -# hline -fbuf.fill(0) -fbuf.hline(0, 1, w, 1) -print('hline', buf) + # hline + fbuf.fill(0) + fbuf.hline(0, 1, w, 1) + print('hline', buf) -# vline -fbuf.fill(0) -fbuf.vline(1, 0, h, 1) -print('vline', buf) + # vline + fbuf.fill(0) + fbuf.vline(1, 0, h, 1) + print('vline', buf) -# rect -fbuf.fill(0) -fbuf.rect(1, 1, 3, 3, 1) -print('rect', buf) + # rect + fbuf.fill(0) + fbuf.rect(1, 1, 3, 3, 1) + print('rect', buf) -#fill rect -fbuf.fill(0) -fbuf.fill_rect(0, 0, 0, 3, 1) # zero width, no-operation -fbuf.fill_rect(1, 1, 3, 3, 1) -print('fill_rect', buf) + #fill rect + fbuf.fill(0) + fbuf.fill_rect(0, 0, 0, 3, 1) # zero width, no-operation + fbuf.fill_rect(1, 1, 3, 3, 1) + print('fill_rect', buf) -# line -fbuf.fill(0) -fbuf.line(1, 1, 3, 3, 1) -print('line', buf) + # line + fbuf.fill(0) + fbuf.line(1, 1, 3, 3, 1) + print('line', buf) -# line steep negative gradient -fbuf.fill(0) -fbuf.line(3, 3, 2, 1, 1) -print('line', buf) + # line steep negative gradient + fbuf.fill(0) + fbuf.line(3, 3, 2, 1, 1) + print('line', buf) -# scroll -fbuf.fill(0) -fbuf.pixel(2, 7, 1) -fbuf.scroll(0, 1) -print(buf) -fbuf.scroll(0, -2) -print(buf) -fbuf.scroll(1, 0) -print(buf) -fbuf.scroll(-1, 0) -print(buf) -fbuf.scroll(2, 2) -print(buf) + # scroll + fbuf.fill(0) + fbuf.pixel(2, 7, 1) + fbuf.scroll(0, 1) + print(buf) + fbuf.scroll(0, -2) + print(buf) + fbuf.scroll(1, 0) + print(buf) + fbuf.scroll(-1, 0) + print(buf) + fbuf.scroll(2, 2) + print(buf) -# print text -fbuf.fill(0) -fbuf.text("hello", 0, 0, 1) -print(buf) -fbuf.text("hello", 0, 0, 0) # clear -print(buf) + # print text + fbuf.fill(0) + fbuf.text("hello", 0, 0, 1) + print(buf) + fbuf.text("hello", 0, 0, 0) # clear + print(buf) -# char out of font range set to chr(127) -fbuf.text(str(chr(31)), 0, 0) -print(buf) + # char out of font range set to chr(127) + fbuf.text(str(chr(31)), 0, 0) + print(buf) + print() # test invalid constructor, and stride argument try: diff --git a/tests/extmod/framebuf1.py.exp b/tests/extmod/framebuf1.py.exp index 83d775d3c4..736ad7a454 100644 --- a/tests/extmod/framebuf1.py.exp +++ b/tests/extmod/framebuf1.py.exp @@ -1,3 +1,4 @@ +MVLSB 0 bytearray(b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff') bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') @@ -18,4 +19,49 @@ bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01') bytearray(b'\x00\x7f\x7f\x04\x04\x00\x00\x00\x00\x00') bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') bytearray(b'\xaaU\xaaU\xaa\x00\x00\x00\x00\x00') + +MHLSB +0 +bytearray(b'\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8\xf8') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') +bytearray(b'\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00') +bytearray(b'\x88\x00\x00\x00\x00\x00\x00\x00\x00\x00') +1 0 +hline bytearray(b'\x00\xf8\x00\x00\x00\x00\x00\x00\x00\x00') +vline bytearray(b'@@@@@@@@@@') +rect bytearray(b'\x00pPp\x00\x00\x00\x00\x00\x00') +fill_rect bytearray(b'\x00ppp\x00\x00\x00\x00\x00\x00') +line bytearray(b'\x00@ \x10\x00\x00\x00\x00\x00\x00') +line bytearray(b'\x00 \x10\x00\x00\x00\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00 \x00') +bytearray(b'\x00\x00\x00\x00\x00\x00 \x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x10\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00 \x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x08\x00') +bytearray(b'``x````\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') +bytearray(b'P\xa8P\xa8P\xa8P\xa8\x00\x00') + +MHMSB +0 +bytearray(b'\x1f\x1f\x1f\x1f\x1f\x1f\x1f\x1f\x1f\x1f') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') +bytearray(b'\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00') +bytearray(b'\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00') +1 0 +hline bytearray(b'\x00\x1f\x00\x00\x00\x00\x00\x00\x00\x00') +vline bytearray(b'\x02\x02\x02\x02\x02\x02\x02\x02\x02\x02') +rect bytearray(b'\x00\x0e\n\x0e\x00\x00\x00\x00\x00\x00') +fill_rect bytearray(b'\x00\x0e\x0e\x0e\x00\x00\x00\x00\x00\x00') +line bytearray(b'\x00\x02\x04\x08\x00\x00\x00\x00\x00\x00') +line bytearray(b'\x00\x04\x04\x08\x00\x00\x00\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x08\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00') +bytearray(b'\x06\x06\x1e\x06\x06\x06\x06\x00\x00\x00') +bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') +bytearray(b'\n\x15\n\x15\n\x15\n\x15\x00\x00') + ValueError From 1a5c8d1053b391ea0ce260b882a1f1026ee7a6e0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Mar 2017 18:42:27 +1100 Subject: [PATCH 016/166] py/vm: Don't release the GIL if the scheduler is locked. The scheduler being locked general means we are running a scheduled function, and switching to another thread violates that, so don't switch in such a case (even though we technically could). And if we are running a scheduled function then we want to finish it ASAP, so we shouldn't switch to another thread. Furthermore, ports with threading enabled will lock the scheduler during a hard IRQ, and this patch to the VM will make sure that threads are not switched during a hard IRQ (which would crash the VM). --- py/vm.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/py/vm.c b/py/vm.c index 98eababc04..ed8b9ec4da 100644 --- a/py/vm.c +++ b/py/vm.c @@ -1301,8 +1301,14 @@ pending_exception_check: #else { #endif + #if MICROPY_ENABLE_SCHEDULER + // can only switch threads if the scheduler is unlocked + if (MP_STATE_VM(sched_state) == MP_SCHED_IDLE) + #endif + { MP_THREAD_GIL_EXIT(); MP_THREAD_GIL_ENTER(); + } } #endif From ebbaf7ee57a7c70067b8d1cc1d148cbdaf14762f Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Mar 2017 18:56:46 +1100 Subject: [PATCH 017/166] stmhal/pendsv: Disable interrupts during a thread switch. We can actually handle interrupts during a thread switch (because we always have a valid stack), but only if those interrupts don't access any of the thread state (because the state may not correspond to the stack pointer). So to be on the safe side we disable interrupts during the very short period of the thread state+stack switch. --- stmhal/pendsv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stmhal/pendsv.c b/stmhal/pendsv.c index 4c2a14de1b..200d13a5a9 100644 --- a/stmhal/pendsv.c +++ b/stmhal/pendsv.c @@ -106,11 +106,14 @@ void pendsv_isr_handler(void) { ".no_obj:\n" // pendsv_object==NULL "push {r4-r11, lr}\n" "vpush {s16-s31}\n" + "mrs r5, primask\n" // save PRIMASK in r5 + "cpsid i\n" // disable interrupts while we change stacks "mov r0, sp\n" // pass sp to save "mov r4, lr\n" // save lr because we are making a call "bl pyb_thread_next\n" // get next thread to execute "mov lr, r4\n" // restore lr "mov sp, r0\n" // switch stacks + "msr primask, r5\n" // reenable interrupts "vpop {s16-s31}\n" "pop {r4-r11, lr}\n" "bx lr\n" // return from interrupt; will return to new thread From b16c35486fa45b7828248075a447690701b1399d Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 21 Mar 2017 15:13:15 +1100 Subject: [PATCH 018/166] esp8266/machine_pin: Fix memset size for zeroing of pin_irq_is_hard. Thanks to @robert-hh. --- esp8266/machine_pin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp8266/machine_pin.c b/esp8266/machine_pin.c index f2e334b1ae..d602bd69f9 100644 --- a/esp8266/machine_pin.c +++ b/esp8266/machine_pin.c @@ -95,7 +95,7 @@ void pin_init0(void) { ETS_GPIO_INTR_ATTACH(pin_intr_handler_iram, NULL); // disable all interrupts memset(&MP_STATE_PORT(pin_irq_handler)[0], 0, 16 * sizeof(mp_obj_t)); - memset(pin_irq_is_hard, 0, sizeof(pin_irq_obj)); + memset(pin_irq_is_hard, 0, sizeof(pin_irq_is_hard)); for (int p = 0; p < 16; ++p) { GPIO_REG_WRITE(GPIO_STATUS_W1TC_ADDRESS, 1 << p); SET_TRIGGER(p, 0); From 5d05ff140626c369452f32de841aa1789e490835 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 21 Mar 2017 15:28:31 +1100 Subject: [PATCH 019/166] esp8266/machine_pin: Fix pin.irq() to work when all args are keywords. --- esp8266/machine_pin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp8266/machine_pin.c b/esp8266/machine_pin.c index d602bd69f9..d4c6d3dea7 100644 --- a/esp8266/machine_pin.c +++ b/esp8266/machine_pin.c @@ -368,7 +368,7 @@ STATIC mp_obj_t pyb_pin_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_t *k nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "pin does not have IRQ capabilities")); } - if (n_args > 1) { + if (n_args > 1 || kw_args->used != 0) { // configure irq mp_obj_t handler = args[ARG_handler].u_obj; uint32_t trigger = args[ARG_trigger].u_int; From 080210ddc60cdfcb33992c20104dd3b2b817861e Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Mar 2017 12:39:32 +1100 Subject: [PATCH 020/166] stmhal/irq: Shift IRQ priorities of TIM and EXTINT to be above PENDSV. This way, Timer and ExtInt callbacks can interrupt the low-priority pendsv handler (for example thread switching). --- stmhal/irq.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stmhal/irq.h b/stmhal/irq.h index bb1749e031..a56f23652e 100644 --- a/stmhal/irq.h +++ b/stmhal/irq.h @@ -133,10 +133,10 @@ MP_DECLARE_CONST_FUN_OBJ_0(pyb_irq_stats_obj); #define IRQ_SUBPRI_CAN 0 // Interrupt priority for non-special timers. -#define IRQ_PRI_TIMX 14 +#define IRQ_PRI_TIMX 13 #define IRQ_SUBPRI_TIMX 0 -#define IRQ_PRI_EXTINT 15 +#define IRQ_PRI_EXTINT 14 #define IRQ_SUBPRI_EXTINT 0 // PENDSV should be at the lowst priority so that other interrupts complete From 96c35d0ac4b63dfbb746b6391f0850a923ed87d6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Mar 2017 12:44:04 +1100 Subject: [PATCH 021/166] stmhal/pybthread: Allow interrupts to work during lock/unlock of mutex. When locking/unlocking a mutex we only need to protect against a thread switch, not general interrupts. --- stmhal/pybthread.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/stmhal/pybthread.c b/stmhal/pybthread.c index 51e9a738d0..6baf88f66b 100644 --- a/stmhal/pybthread.c +++ b/stmhal/pybthread.c @@ -37,6 +37,11 @@ #define PYB_MUTEX_UNLOCKED ((void*)0) #define PYB_MUTEX_LOCKED ((void*)1) +// These macros are used when we only need to protect against a thread +// switch; other interrupts are still allowed to proceed. +#define RAISE_IRQ_PRI() raise_irq_pri(IRQ_PRI_PENDSV) +#define RESTORE_IRQ_PRI(state) restore_irq_pri(state) + extern void __fatal_error(const char*); volatile int pyb_thread_enabled; @@ -176,15 +181,15 @@ void pyb_mutex_init(pyb_mutex_t *m) { } int pyb_mutex_lock(pyb_mutex_t *m, int wait) { - uint32_t irq_state = disable_irq(); + uint32_t irq_state = RAISE_IRQ_PRI(); if (*m == PYB_MUTEX_UNLOCKED) { // mutex is available *m = PYB_MUTEX_LOCKED; - enable_irq(irq_state); + RESTORE_IRQ_PRI(irq_state); } else { // mutex is locked if (!wait) { - enable_irq(irq_state); + RESTORE_IRQ_PRI(irq_state); return 0; // failed to lock mutex } if (*m == PYB_MUTEX_LOCKED) { @@ -202,14 +207,14 @@ int pyb_mutex_lock(pyb_mutex_t *m, int wait) { pyb_thread_remove_from_runable(pyb_thread_cur); // thread switch will occur after we enable irqs SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; - enable_irq(irq_state); + RESTORE_IRQ_PRI(irq_state); // when we come back we have the mutex } return 1; // have mutex } void pyb_mutex_unlock(pyb_mutex_t *m) { - uint32_t irq_state = disable_irq(); + uint32_t irq_state = RAISE_IRQ_PRI(); if (*m == PYB_MUTEX_LOCKED) { // no threads are blocked on the mutex *m = PYB_MUTEX_UNLOCKED; @@ -226,7 +231,7 @@ void pyb_mutex_unlock(pyb_mutex_t *m) { // put unblocked thread on runable list pyb_thread_add_to_runable(th); } - enable_irq(irq_state); + RESTORE_IRQ_PRI(irq_state); } #endif // MICROPY_PY_THREAD From 2e3fc778094c0c31c3e15e196e1c7b3b838239fe Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Mar 2017 12:49:21 +1100 Subject: [PATCH 022/166] extmod/utime_mphal: Don't exit/enter the GIL in generic sleep functions. GIL behaviour should be handled by the port. And ports probably want to define sleep_us so that it doesn't release the GIL, to improve timing accuracy. --- extmod/utime_mphal.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/extmod/utime_mphal.c b/extmod/utime_mphal.c index f447b3a686..e99ba46ce5 100644 --- a/extmod/utime_mphal.c +++ b/extmod/utime_mphal.c @@ -37,13 +37,11 @@ #include "extmod/utime_mphal.h" STATIC mp_obj_t time_sleep(mp_obj_t seconds_o) { - MP_THREAD_GIL_EXIT(); #if MICROPY_PY_BUILTINS_FLOAT mp_hal_delay_ms(1000 * mp_obj_get_float(seconds_o)); #else mp_hal_delay_ms(1000 * mp_obj_get_int(seconds_o)); #endif - MP_THREAD_GIL_ENTER(); return mp_const_none; } MP_DEFINE_CONST_FUN_OBJ_1(mp_utime_sleep_obj, time_sleep); @@ -51,9 +49,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_utime_sleep_obj, time_sleep); STATIC mp_obj_t time_sleep_ms(mp_obj_t arg) { mp_int_t ms = mp_obj_get_int(arg); if (ms > 0) { - MP_THREAD_GIL_EXIT(); mp_hal_delay_ms(ms); - MP_THREAD_GIL_ENTER(); } return mp_const_none; } @@ -62,9 +58,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_utime_sleep_ms_obj, time_sleep_ms); STATIC mp_obj_t time_sleep_us(mp_obj_t arg) { mp_int_t us = mp_obj_get_int(arg); if (us > 0) { - MP_THREAD_GIL_EXIT(); mp_hal_delay_us(us); - MP_THREAD_GIL_ENTER(); } return mp_const_none; } From 3509e2d3074486e426b41aa89c6e01d591eb01ad Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Mar 2017 12:54:43 +1100 Subject: [PATCH 023/166] stmhal/systick: Make mp_hal_delay_ms release the GIL when sleeping. --- stmhal/systick.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/stmhal/systick.c b/stmhal/systick.c index 71e3d34889..4eac583e2a 100644 --- a/stmhal/systick.c +++ b/stmhal/systick.c @@ -51,24 +51,17 @@ void HAL_Delay(uint32_t Delay) { } // Core delay function that does an efficient sleep and may switch thread context. -// Note: Upon entering this function we may or may not have the GIL. +// If IRQs are enabled then we must have the GIL. void mp_hal_delay_ms(mp_uint_t Delay) { if (query_irq() == IRQ_STATE_ENABLED) { // IRQs enabled, so can use systick counter to do the delay uint32_t start = uwTick; // Wraparound of tick is taken care of by 2's complement arithmetic. while (uwTick - start < Delay) { - // Enter sleep mode, waiting for (at least) the SysTick interrupt. - mp_handle_pending(); - #if MICROPY_PY_THREAD - if (pyb_thread_enabled) { - pyb_thread_yield(); - } else { - __WFI(); - } - #else - __WFI(); - #endif + // This macro will execute the necessary idle behaviour. It may + // raise an exception, switch threads or enter sleep mode (waiting for + // (at least) the SysTick interrupt). + MICROPY_EVENT_POLL_HOOK } } else { // IRQs disabled, so need to use a busy loop for the delay. From 1110c8873c984f3abf9ee6ceff12ca87a45dd238 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Mar 2017 12:57:51 +1100 Subject: [PATCH 024/166] cc3200/mods/modutime: Use generic sleep_ms and sleep_us implementations. --- cc3200/mods/modutime.c | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/cc3200/mods/modutime.c b/cc3200/mods/modutime.c index de1f750124..0aa853dd67 100644 --- a/cc3200/mods/modutime.c +++ b/cc3200/mods/modutime.c @@ -131,24 +131,6 @@ STATIC mp_obj_t time_sleep(mp_obj_t seconds_o) { } MP_DEFINE_CONST_FUN_OBJ_1(time_sleep_obj, time_sleep); -STATIC mp_obj_t time_sleep_ms (mp_obj_t ms_in) { - mp_int_t ms = mp_obj_get_int(ms_in); - if (ms > 0) { - mp_hal_delay_ms(ms); - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(time_sleep_ms_obj, time_sleep_ms); - -STATIC mp_obj_t time_sleep_us (mp_obj_t usec_in) { - mp_int_t usec = mp_obj_get_int(usec_in); - if (usec > 0) { - mp_hal_delay_us(usec); - } - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(time_sleep_us_obj, time_sleep_us); - STATIC const mp_map_elem_t time_module_globals_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR_utime) }, @@ -158,8 +140,8 @@ STATIC const mp_map_elem_t time_module_globals_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR_sleep), (mp_obj_t)&time_sleep_obj }, // MicroPython additions - { MP_OBJ_NEW_QSTR(MP_QSTR_sleep_ms), (mp_obj_t)&time_sleep_ms_obj }, - { MP_OBJ_NEW_QSTR(MP_QSTR_sleep_us), (mp_obj_t)&time_sleep_us_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_sleep_ms), (mp_obj_t)&mp_utime_sleep_ms_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_sleep_us), (mp_obj_t)&mp_utime_sleep_us_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_ticks_ms), (mp_obj_t)&mp_utime_ticks_ms_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_ticks_us), (mp_obj_t)&mp_utime_ticks_us_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_ticks_cpu), (mp_obj_t)&mp_utime_ticks_cpu_obj }, From 58f23def55a705357b352ce8af642e03cd5c49f0 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Mar 2017 13:40:27 +1100 Subject: [PATCH 025/166] py/bc: Provide better error message for an unexpected keyword argument. Now, passing a keyword argument that is not expected will correctly report that fact. If normal or detailed error messages are enabled then the name of the unexpected argument will be reported. This patch decreases the code size of bare-arm and stmhal by 12 bytes, and cc3200 by 8 bytes. Other ports (minimal, unix, esp8266) remain the same in code size. For terse error message configuration this is because the new message is shorter than the old one. For normal (and detailed) error message configuration this is because the new error message already exists in py/objnamedtuple.c so there's no extra space in ROM needed for the string. --- py/bc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/py/bc.c b/py/bc.c index e7a1a333f7..12887645b9 100644 --- a/py/bc.c +++ b/py/bc.c @@ -196,7 +196,12 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw } // Didn't find name match with positional args if ((scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) == 0) { - mp_raise_msg(&mp_type_TypeError, "function does not take keyword arguments"); + if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { + mp_raise_msg(&mp_type_TypeError, "unexpected keyword argument"); + } else { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, + "unexpected keyword argument '%q'", MP_OBJ_QSTR_VALUE(wanted_arg_name))); + } } mp_obj_dict_store(dict, kwargs[2 * i], kwargs[2 * i + 1]); continue2:; From 4a4bb84e9216a130050a13e20a19fbf476ed6e6c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 22 Mar 2017 22:17:52 +0300 Subject: [PATCH 026/166] tests/heapalloc_str: Test no-replacement case for str.replace(). --- tests/micropython/heapalloc_str.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/micropython/heapalloc_str.py b/tests/micropython/heapalloc_str.py index 0df39d3c0c..39aa56ccd7 100644 --- a/tests/micropython/heapalloc_str.py +++ b/tests/micropython/heapalloc_str.py @@ -3,6 +3,7 @@ import micropython micropython.heap_lock() +# Concatenating empty string returns original string b"" + b"" b"" + b"1" b"2" + b"" @@ -11,4 +12,7 @@ b"2" + b"" "" + "1" "2" + "" +# If no replacements done, returns original string +"foo".replace(",", "_") + micropython.heap_unlock() From 75589272ef12d538ab7ce4f4453be85d826b5083 Mon Sep 17 00:00:00 2001 From: Krzysztof Blazewicz Date: Sun, 5 Mar 2017 13:28:27 +0100 Subject: [PATCH 027/166] all/Makefile: Remove -ansi from GCC flags, its ignored anyway. The -ansi flag is used for C dialect selection and it is equivalent to -std=c90. Because it goes right before -std=gnu99 it is ignored as for conflicting flags GCC always uses the last one. --- bare-arm/Makefile | 2 +- cc3200/Makefile | 2 +- esp8266/Makefile | 2 +- examples/embedding/Makefile.upylib | 2 +- minimal/Makefile | 4 ++-- mpy-cross/Makefile | 2 +- pic16bit/Makefile | 2 +- qemu-arm/Makefile | 2 +- stmhal/Makefile | 2 +- teensy/Makefile | 2 +- unix/Makefile | 2 +- windows/Makefile | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bare-arm/Makefile b/bare-arm/Makefile index a57ccfd196..6ee8552628 100644 --- a/bare-arm/Makefile +++ b/bare-arm/Makefile @@ -13,7 +13,7 @@ INC += -I.. INC += -I$(BUILD) CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -mabi=aapcs-linux -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion -CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) +CFLAGS = $(INC) -Wall -Werror -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) #Debugging/Optimization ifeq ($(DEBUG), 1) diff --git a/cc3200/Makefile b/cc3200/Makefile index 4db3b9e8d3..663496435b 100644 --- a/cc3200/Makefile +++ b/cc3200/Makefile @@ -20,7 +20,7 @@ include ../py/mkenv.mk CROSS_COMPILE ?= arm-none-eabi- CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -march=armv7e-m -mabi=aapcs -mcpu=cortex-m4 -msoft-float -mfloat-abi=soft -fsingle-precision-constant -Wdouble-promotion -CFLAGS = -Wall -Wpointer-arith -Werror -ansi -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) -Os +CFLAGS = -Wall -Wpointer-arith -Werror -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) -Os CFLAGS += -g -ffunction-sections -fdata-sections -fno-common -fsigned-char -mno-unaligned-access CFLAGS += -Iboards/$(BOARD) CFLAGS += $(CFLAGS_MOD) diff --git a/esp8266/Makefile b/esp8266/Makefile index 5e61fc16d8..4710f4cdc2 100644 --- a/esp8266/Makefile +++ b/esp8266/Makefile @@ -41,7 +41,7 @@ CFLAGS_XTENSA = -fsingle-precision-constant -Wdouble-promotion \ -Wl,-EL -mlongcalls -mtext-section-literals -mforce-l32 \ -DLWIP_OPEN_SRC -CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -ansi -std=gnu99 -nostdlib -DUART_OS=$(UART_OS) \ +CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -std=gnu99 -nostdlib -DUART_OS=$(UART_OS) \ $(CFLAGS_XTENSA) $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) LDSCRIPT = esp8266.ld diff --git a/examples/embedding/Makefile.upylib b/examples/embedding/Makefile.upylib index d8dbade3fe..8b506e95e7 100644 --- a/examples/embedding/Makefile.upylib +++ b/examples/embedding/Makefile.upylib @@ -20,7 +20,7 @@ INC += -I$(BUILD) # compiler settings CWARN = -Wall -Werror CWARN += -Wpointer-arith -Wuninitialized -CFLAGS = $(INC) $(CWARN) -ansi -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) +CFLAGS = $(INC) $(CWARN) -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) # Debugging/Optimization ifdef DEBUG diff --git a/minimal/Makefile b/minimal/Makefile index 4117517352..42c2d53127 100644 --- a/minimal/Makefile +++ b/minimal/Makefile @@ -22,9 +22,9 @@ ifeq ($(CROSS), 1) DFU = ../tools/dfu.py PYDFU = ../tools/pydfu.py CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -mabi=aapcs-linux -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion -CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) +CFLAGS = $(INC) -Wall -Werror -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) else -CFLAGS = -m32 $(INC) -Wall -Werror -ansi -std=gnu99 $(COPT) +CFLAGS = -m32 $(INC) -Wall -Werror -std=gnu99 $(COPT) endif #Debugging/Optimization diff --git a/mpy-cross/Makefile b/mpy-cross/Makefile index 3f99566209..3d5e5da2f9 100644 --- a/mpy-cross/Makefile +++ b/mpy-cross/Makefile @@ -19,7 +19,7 @@ INC += -I$(BUILD) # compiler settings CWARN = -Wall -Werror CWARN += -Wpointer-arith -Wuninitialized -CFLAGS = $(INC) $(CWARN) -ansi -std=gnu99 $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) +CFLAGS = $(INC) $(CWARN) -std=gnu99 $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) CFLAGS += -fdata-sections -ffunction-sections -fno-asynchronous-unwind-tables # Debugging/Optimization diff --git a/pic16bit/Makefile b/pic16bit/Makefile index cfe8af86ec..add9a8495c 100644 --- a/pic16bit/Makefile +++ b/pic16bit/Makefile @@ -21,7 +21,7 @@ INC += -I$(XC16)/include INC += -I$(XC16)/support/$(PARTFAMILY)/h CFLAGS_PIC16BIT = -mcpu=$(PART) -mlarge-code -CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -nostdlib $(CFLAGS_PIC16BIT) $(COPT) +CFLAGS = $(INC) -Wall -Werror -std=gnu99 -nostdlib $(CFLAGS_PIC16BIT) $(COPT) #Debugging/Optimization ifeq ($(DEBUG), 1) diff --git a/qemu-arm/Makefile b/qemu-arm/Makefile index 61fc243642..e8f5b359ef 100644 --- a/qemu-arm/Makefile +++ b/qemu-arm/Makefile @@ -15,7 +15,7 @@ INC += -I$(BUILD) INC += -I../tools/tinytest/ CFLAGS_CORTEX_M3 = -mthumb -mcpu=cortex-m3 -mfloat-abi=soft -CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -ansi -std=gnu99 $(CFLAGS_CORTEX_M3) $(COPT) \ +CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -std=gnu99 $(CFLAGS_CORTEX_M3) $(COPT) \ -ffunction-sections -fdata-sections #Debugging/Optimization diff --git a/stmhal/Makefile b/stmhal/Makefile index 76579d1305..3eef6ffb23 100644 --- a/stmhal/Makefile +++ b/stmhal/Makefile @@ -56,7 +56,7 @@ CFLAGS_MCU_f4 = $(CFLAGS_CORTEX_M) -mtune=cortex-m4 -mcpu=cortex-m4 -DMCU_SERIES CFLAGS_MCU_f7 = $(CFLAGS_CORTEX_M) -mtune=cortex-m7 -mcpu=cortex-m7 -DMCU_SERIES_F7 CFLAGS_MCU_l4 = $(CFLAGS_CORTEX_M) -mtune=cortex-m4 -mcpu=cortex-m4 -DMCU_SERIES_L4 -CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -ansi -std=gnu99 -nostdlib $(CFLAGS_MOD) +CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -std=gnu99 -nostdlib $(CFLAGS_MOD) CFLAGS += -D$(CMSIS_MCU) CFLAGS += $(CFLAGS_MCU_$(MCU_SERIES)) CFLAGS += $(COPT) diff --git a/teensy/Makefile b/teensy/Makefile index 82e04a1ae3..e613b8f271 100644 --- a/teensy/Makefile +++ b/teensy/Makefile @@ -36,7 +36,7 @@ INC += -I../lib/mp-readline INC += -I$(BUILD) INC += -Icore -CFLAGS = $(INC) -Wall -Wpointer-arith -ansi -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) +CFLAGS = $(INC) -Wall -Wpointer-arith -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) LDFLAGS = -nostdlib -T mk20dx256.ld -msoft-float -mfloat-abi=soft ifeq ($(USE_ARDUINO_TOOLCHAIN),1) diff --git a/unix/Makefile b/unix/Makefile index 83bd3f984c..f28a4bcae9 100644 --- a/unix/Makefile +++ b/unix/Makefile @@ -23,7 +23,7 @@ INC += -I$(BUILD) # compiler settings CWARN = -Wall -Werror CWARN += -Wpointer-arith -Wuninitialized -CFLAGS = $(INC) $(CWARN) -ansi -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) +CFLAGS = $(INC) $(CWARN) -std=gnu99 -DUNIX $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA) # Debugging/Optimization ifdef DEBUG diff --git a/windows/Makefile b/windows/Makefile index 68de66a071..72c97381bd 100644 --- a/windows/Makefile +++ b/windows/Makefile @@ -15,7 +15,7 @@ INC += -I.. INC += -I$(BUILD) # compiler settings -CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -ansi -std=gnu99 -DUNIX -D__USE_MINGW_ANSI_STDIO=1 $(CFLAGS_MOD) $(COPT) +CFLAGS = $(INC) -Wall -Wpointer-arith -Werror -std=gnu99 -DUNIX -D__USE_MINGW_ANSI_STDIO=1 $(CFLAGS_MOD) $(COPT) LDFLAGS = $(LDFLAGS_MOD) -lm # Debugging/Optimization From 4afa782fb415165aa6e7f9eea8532f0d2493f8ce Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 23 Mar 2017 15:41:04 +1100 Subject: [PATCH 028/166] bare-arm/Makefile: Change C standard from gnu99 to c99. --- bare-arm/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bare-arm/Makefile b/bare-arm/Makefile index 6ee8552628..cfd427c609 100644 --- a/bare-arm/Makefile +++ b/bare-arm/Makefile @@ -13,7 +13,7 @@ INC += -I.. INC += -I$(BUILD) CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -mabi=aapcs-linux -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion -CFLAGS = $(INC) -Wall -Werror -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) +CFLAGS = $(INC) -Wall -Werror -std=c99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) #Debugging/Optimization ifeq ($(DEBUG), 1) From 92cd0008428d5db6e8cf3ae531cdb24acdfb68e1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 23 Mar 2017 15:41:38 +1100 Subject: [PATCH 029/166] minimal/Makefile: Change C standard from gnu99 to c99. --- minimal/Makefile | 4 ++-- minimal/main.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/minimal/Makefile b/minimal/Makefile index 42c2d53127..3b446beae7 100644 --- a/minimal/Makefile +++ b/minimal/Makefile @@ -22,9 +22,9 @@ ifeq ($(CROSS), 1) DFU = ../tools/dfu.py PYDFU = ../tools/pydfu.py CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -mabi=aapcs-linux -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion -CFLAGS = $(INC) -Wall -Werror -std=gnu99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) +CFLAGS = $(INC) -Wall -Werror -std=c99 -nostdlib $(CFLAGS_CORTEX_M4) $(COPT) else -CFLAGS = -m32 $(INC) -Wall -Werror -std=gnu99 $(COPT) +CFLAGS = -m32 $(INC) -Wall -Werror -std=c99 $(COPT) endif #Debugging/Optimization diff --git a/minimal/main.c b/minimal/main.c index 496d925e77..5cc88ff668 100644 --- a/minimal/main.c +++ b/minimal/main.c @@ -106,7 +106,7 @@ extern uint32_t _estack, _sidata, _sdata, _edata, _sbss, _ebss; void Reset_Handler(void) __attribute__((naked)); void Reset_Handler(void) { // set stack pointer - asm volatile ("ldr sp, =_estack"); + __asm volatile ("ldr sp, =_estack"); // copy .data section from flash to RAM for (uint32_t *src = &_sidata, *dest = &_sdata; dest < &_edata;) { *dest++ = *src++; From c61131380da73221a6590f7898d68b35a8450891 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 20 Mar 2017 16:15:46 +0100 Subject: [PATCH 030/166] windows: Make msvc project file support any version from VS2013 to VS2017 Instead of having the PlatformToolset property hardcoded to a specific version just set it to the value of DefaultPlatformToolset: this gets defined according to the commandline environment in which the build was started. Instead of just supporting VS2015 the project can now be built by any version from VS2013 to VS2017 and normally future versions as well, without quirks like VS asking whether you want to upgrade the project to the latest version (as was the case when opening the project in VS2017) or not being able to build at all (as was the case when opening the project in VS2013). Also adjust the .gitignore file to ignore any artefacts from VS2017. --- windows/.gitignore | 1 + windows/micropython.vcxproj | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/windows/.gitignore b/windows/.gitignore index ec28212111..12235e7c9e 100644 --- a/windows/.gitignore +++ b/windows/.gitignore @@ -7,4 +7,5 @@ *.ilk *.filters /build/* +.vs/* *.VC.*db diff --git a/windows/micropython.vcxproj b/windows/micropython.vcxproj index 6f6e11ab4a..ee0b98abba 100644 --- a/windows/micropython.vcxproj +++ b/windows/micropython.vcxproj @@ -26,26 +26,26 @@ Application true - v140 + $(DefaultPlatformToolset) MultiByte Application false - v140 + $(DefaultPlatformToolset) true MultiByte Application true - v140 + $(DefaultPlatformToolset) MultiByte Application false - v140 + $(DefaultPlatformToolset) true MultiByte From 3f3df435014cc835ce9357e15ddb8996c5866f43 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 22 Mar 2017 10:54:40 +0100 Subject: [PATCH 031/166] msvc: Remove directory with generated files when cleaning. This assures after cleaning all build artefacts (qstr related files, generated version header) have been removed. --- windows/msvc/genhdr.targets | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/windows/msvc/genhdr.targets b/windows/msvc/genhdr.targets index 8c2ba8eb21..afe5f5d765 100644 --- a/windows/msvc/genhdr.targets +++ b/windows/msvc/genhdr.targets @@ -87,6 +87,10 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) { + + + +