From a25aa2bcc3348349d8b1deb8fcdd80b3f10d0550 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 8 Feb 2017 10:48:51 +1100 Subject: [PATCH 01/92] py/asmxtensa.h: Explicitly cast args to 32-bits so left-shift is legal. For archs that have 16-bit pointers, the asmxtensa.h file can give compiler warnings about left-shift being greater than the width of the type (due to the inline functions in this header file). Explicitly casting the constants to uint32_t stops these warnings. --- py/asmxtensa.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/asmxtensa.h b/py/asmxtensa.h index 12083252eb..7db6c0d3dc 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -73,11 +73,11 @@ // macros for encoding instructions (little endian versions) #define ASM_XTENSA_ENCODE_RRR(op0, op1, op2, r, s, t) \ - (((op2) << 20) | ((op1) << 16) | ((r) << 12) | ((s) << 8) | ((t) << 4) | (op0)) + ((((uint32_t)op2) << 20) | (((uint32_t)op1) << 16) | ((r) << 12) | ((s) << 8) | ((t) << 4) | (op0)) #define ASM_XTENSA_ENCODE_RRI4(op0, op1, r, s, t, imm4) \ (((imm4) << 20) | ((op1) << 16) | ((r) << 12) | ((s) << 8) | ((t) << 4) | (op0)) #define ASM_XTENSA_ENCODE_RRI8(op0, r, s, t, imm8) \ - (((imm8) << 16) | ((r) << 12) | ((s) << 8) | ((t) << 4) | (op0)) + ((((uint32_t)imm8) << 16) | ((r) << 12) | ((s) << 8) | ((t) << 4) | (op0)) #define ASM_XTENSA_ENCODE_RI16(op0, t, imm16) \ (((imm16) << 8) | ((t) << 4) | (op0)) #define ASM_XTENSA_ENCODE_RSR(op0, op1, op2, rs, t) \ @@ -85,7 +85,7 @@ #define ASM_XTENSA_ENCODE_CALL(op0, n, offset) \ (((offset) << 6) | ((n) << 4) | (op0)) #define ASM_XTENSA_ENCODE_CALLX(op0, op1, op2, r, s, m, n) \ - (((op2) << 20) | ((op1) << 16) | ((r) << 12) | ((s) << 8) | ((m) << 6) | ((n) << 4) | (op0)) + ((((uint32_t)op2) << 20) | (((uint32_t)op1) << 16) | ((r) << 12) | ((s) << 8) | ((m) << 6) | ((n) << 4) | (op0)) #define ASM_XTENSA_ENCODE_BRI8(op0, r, s, m, n, imm8) \ (((imm8) << 16) | ((r) << 12) | ((s) << 8) | ((m) << 6) | ((n) << 4) | (op0)) #define ASM_XTENSA_ENCODE_BRI12(op0, s, m, n, imm12) \ From af622eb2a6aa50429eb64716e754ee58c3f87c2e Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 8 Feb 2017 11:00:15 +1100 Subject: [PATCH 02/92] py/map: Change mp_uint_t to size_t where appropriate. The internal map/set functions now use size_t exclusively for computing addresses. size_t is enough to reach all of available memory when computing addresses so is the right type to use. In particular, for nanbox builds it saves quite a bit of code size and RAM compared to the original use of mp_uint_t (which is 64-bits on nanbox builds). --- py/map.c | 32 ++++++++++++++++---------------- py/obj.h | 26 +++++++++++++------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/py/map.c b/py/map.c index 0916ec522d..c45f71e17c 100644 --- a/py/map.c +++ b/py/map.c @@ -56,7 +56,7 @@ STATIC const uint16_t hash_allocation_sizes[] = { 3229, 4831, 7243, 10861, 16273, 24407, 36607, 54907, // *1.5 }; -STATIC mp_uint_t get_hash_alloc_greater_or_equal_to(mp_uint_t x) { +STATIC size_t get_hash_alloc_greater_or_equal_to(size_t x) { for (size_t i = 0; i < MP_ARRAY_SIZE(hash_allocation_sizes); i++) { if (hash_allocation_sizes[i] >= x) { return hash_allocation_sizes[i]; @@ -70,7 +70,7 @@ STATIC mp_uint_t get_hash_alloc_greater_or_equal_to(mp_uint_t x) { /******************************************************************************/ /* map */ -void mp_map_init(mp_map_t *map, mp_uint_t n) { +void mp_map_init(mp_map_t *map, size_t n) { if (n == 0) { map->alloc = 0; map->table = NULL; @@ -84,7 +84,7 @@ void mp_map_init(mp_map_t *map, mp_uint_t n) { map->is_ordered = 0; } -void mp_map_init_fixed_table(mp_map_t *map, mp_uint_t n, const mp_obj_t *table) { +void mp_map_init_fixed_table(mp_map_t *map, size_t n, const mp_obj_t *table) { map->alloc = n; map->used = n; map->all_keys_are_qstrs = 1; @@ -93,7 +93,7 @@ void mp_map_init_fixed_table(mp_map_t *map, mp_uint_t n, const mp_obj_t *table) map->table = (mp_map_elem_t*)table; } -mp_map_t *mp_map_new(mp_uint_t n) { +mp_map_t *mp_map_new(size_t n) { mp_map_t *map = m_new(mp_map_t, 1); mp_map_init(map, n); return map; @@ -124,8 +124,8 @@ void mp_map_clear(mp_map_t *map) { } STATIC void mp_map_rehash(mp_map_t *map) { - mp_uint_t old_alloc = map->alloc; - mp_uint_t new_alloc = get_hash_alloc_greater_or_equal_to(map->alloc + 1); + size_t old_alloc = map->alloc; + size_t new_alloc = get_hash_alloc_greater_or_equal_to(map->alloc + 1); mp_map_elem_t *old_table = map->table; mp_map_elem_t *new_table = m_new0(mp_map_elem_t, new_alloc); // If we reach this point, table resizing succeeded, now we can edit the old map. @@ -133,7 +133,7 @@ STATIC void mp_map_rehash(mp_map_t *map) { map->used = 0; map->all_keys_are_qstrs = 1; map->table = new_table; - for (mp_uint_t i = 0; i < old_alloc; i++) { + for (size_t i = 0; i < old_alloc; i++) { if (old_table[i].key != MP_OBJ_NULL && old_table[i].key != MP_OBJ_SENTINEL) { mp_map_lookup(map, old_table[i].key, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = old_table[i].value; } @@ -220,8 +220,8 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t hash = MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, index)); } - mp_uint_t pos = hash % map->alloc; - mp_uint_t start_pos = pos; + size_t pos = hash % map->alloc; + size_t start_pos = pos; mp_map_elem_t *avail_slot = NULL; for (;;) { mp_map_elem_t *slot = &map->table[pos]; @@ -296,19 +296,19 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t #if MICROPY_PY_BUILTINS_SET -void mp_set_init(mp_set_t *set, mp_uint_t n) { +void mp_set_init(mp_set_t *set, size_t n) { set->alloc = n; set->used = 0; set->table = m_new0(mp_obj_t, set->alloc); } STATIC void mp_set_rehash(mp_set_t *set) { - mp_uint_t old_alloc = set->alloc; + size_t old_alloc = set->alloc; mp_obj_t *old_table = set->table; set->alloc = get_hash_alloc_greater_or_equal_to(set->alloc + 1); set->used = 0; set->table = m_new0(mp_obj_t, set->alloc); - for (mp_uint_t i = 0; i < old_alloc; i++) { + for (size_t i = 0; i < old_alloc; i++) { if (old_table[i] != MP_OBJ_NULL && old_table[i] != MP_OBJ_SENTINEL) { mp_set_lookup(set, old_table[i], MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); } @@ -328,8 +328,8 @@ mp_obj_t mp_set_lookup(mp_set_t *set, mp_obj_t index, mp_map_lookup_kind_t looku } } mp_uint_t hash = MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, index)); - mp_uint_t pos = hash % set->alloc; - mp_uint_t start_pos = pos; + size_t pos = hash % set->alloc; + size_t start_pos = pos; mp_obj_t *avail_slot = NULL; for (;;) { mp_obj_t elem = set->table[pos]; @@ -390,7 +390,7 @@ mp_obj_t mp_set_lookup(mp_set_t *set, mp_obj_t index, mp_map_lookup_kind_t looku } mp_obj_t mp_set_remove_first(mp_set_t *set) { - for (mp_uint_t pos = 0; pos < set->alloc; pos++) { + for (size_t pos = 0; pos < set->alloc; pos++) { if (MP_SET_SLOT_IS_FILLED(set, pos)) { mp_obj_t elem = set->table[pos]; // delete element @@ -418,7 +418,7 @@ void mp_set_clear(mp_set_t *set) { #if defined(DEBUG_PRINT) && DEBUG_PRINT void mp_map_dump(mp_map_t *map) { - for (mp_uint_t i = 0; i < map->alloc; i++) { + for (size_t i = 0; i < map->alloc; i++) { if (map->table[i].key != NULL) { mp_obj_print(map->table[i].key, PRINT_REPR); } else { diff --git a/py/obj.h b/py/obj.h index 6106bbe19a..071e38502b 100644 --- a/py/obj.h +++ b/py/obj.h @@ -353,11 +353,11 @@ typedef struct _mp_rom_map_elem_t { // would also need a trucated dict structure typedef struct _mp_map_t { - mp_uint_t all_keys_are_qstrs : 1; - mp_uint_t is_fixed : 1; // a fixed array that can't be modified; must also be ordered - mp_uint_t is_ordered : 1; // an ordered array - mp_uint_t used : (8 * sizeof(mp_uint_t) - 3); - mp_uint_t alloc; + size_t all_keys_are_qstrs : 1; + size_t is_fixed : 1; // a fixed array that can't be modified; must also be ordered + size_t is_ordered : 1; // an ordered array + size_t used : (8 * sizeof(size_t) - 3); + size_t alloc; mp_map_elem_t *table; } mp_map_t; @@ -371,11 +371,11 @@ typedef enum _mp_map_lookup_kind_t { extern const mp_map_t mp_const_empty_map; -static inline bool MP_MAP_SLOT_IS_FILLED(const mp_map_t *map, mp_uint_t pos) { return ((map)->table[pos].key != MP_OBJ_NULL && (map)->table[pos].key != MP_OBJ_SENTINEL); } +static inline bool MP_MAP_SLOT_IS_FILLED(const mp_map_t *map, size_t pos) { return ((map)->table[pos].key != MP_OBJ_NULL && (map)->table[pos].key != MP_OBJ_SENTINEL); } -void mp_map_init(mp_map_t *map, mp_uint_t n); -void mp_map_init_fixed_table(mp_map_t *map, mp_uint_t n, const mp_obj_t *table); -mp_map_t *mp_map_new(mp_uint_t n); +void mp_map_init(mp_map_t *map, size_t n); +void mp_map_init_fixed_table(mp_map_t *map, size_t n, const mp_obj_t *table); +mp_map_t *mp_map_new(size_t n); void mp_map_deinit(mp_map_t *map); void mp_map_free(mp_map_t *map); mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t lookup_kind); @@ -385,14 +385,14 @@ void mp_map_dump(mp_map_t *map); // Underlying set implementation (not set object) typedef struct _mp_set_t { - mp_uint_t alloc; - mp_uint_t used; + size_t alloc; + size_t used; mp_obj_t *table; } mp_set_t; -static inline bool MP_SET_SLOT_IS_FILLED(const mp_set_t *set, mp_uint_t pos) { return ((set)->table[pos] != MP_OBJ_NULL && (set)->table[pos] != MP_OBJ_SENTINEL); } +static inline bool MP_SET_SLOT_IS_FILLED(const mp_set_t *set, size_t pos) { return ((set)->table[pos] != MP_OBJ_NULL && (set)->table[pos] != MP_OBJ_SENTINEL); } -void mp_set_init(mp_set_t *set, mp_uint_t n); +void mp_set_init(mp_set_t *set, size_t n); mp_obj_t mp_set_lookup(mp_set_t *set, mp_obj_t index, mp_map_lookup_kind_t lookup_kind); mp_obj_t mp_set_remove_first(mp_set_t *set); void mp_set_clear(mp_set_t *set); From aa34c553ec75535f46f5d6ff28c0c943701e569b Mon Sep 17 00:00:00 2001 From: Dave Hylands Date: Tue, 7 Feb 2017 14:22:55 -0800 Subject: [PATCH 03/92] py/nlr: Fix execstack builds for ARM. It seems that the gcc toolchain on the RaspberryPi likes %progbits instead of @progbits. I verified that %progbits also works under x86, so this should fix #2848 and fix #2842 I verified that unix and mpy-cross both compile on my RaspberryPi and on my x64 machine. --- py/nlrx64.S | 2 +- py/nlrx86.S | 2 +- py/nlrxtensa.S | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/py/nlrx64.S b/py/nlrx64.S index fc8b8e3446..78d654558f 100644 --- a/py/nlrx64.S +++ b/py/nlrx64.S @@ -258,5 +258,5 @@ nlr_jump: #endif // defined(__x86_64__) && !MICROPY_NLR_SETJMP #if defined(linux) - .section .note.GNU-stack,"",@progbits + .section .note.GNU-stack,"",%progbits #endif diff --git a/py/nlrx86.S b/py/nlrx86.S index f1de61e11a..ff0cc21305 100644 --- a/py/nlrx86.S +++ b/py/nlrx86.S @@ -191,5 +191,5 @@ nlr_jump: #endif // defined(__i386__) && !MICROPY_NLR_SETJMP #if defined(linux) - .section .note.GNU-stack,"",@progbits + .section .note.GNU-stack,"",%progbits #endif diff --git a/py/nlrxtensa.S b/py/nlrxtensa.S index 73af4832ff..6d87e305d4 100644 --- a/py/nlrxtensa.S +++ b/py/nlrxtensa.S @@ -115,5 +115,5 @@ nlr_jump: #endif // defined(__xtensa__) #if defined(linux) - .section .note.GNU-stack,"",@progbits + .section .note.GNU-stack,"",%progbits #endif From 7ae9bee7905601a827f3d216300f917afbacf07e Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 8 Feb 2017 11:50:22 +1100 Subject: [PATCH 04/92] stmhal/Makefile: Drop use of -mabi=aapcs-linux; link libgcc by default. The aapcs-linux ABI is not required, instead the default aapcs ABI is enough. And using the default ABI means that the provided libgcc will now link with the firmware without warnings about variable vs fixed enums. Although the binary size increases by about 1k, RAM usage is slightly decreased. And libgcc may prove useful in the future for things like long-long division. --- stmhal/Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/stmhal/Makefile b/stmhal/Makefile index a83f2c4957..edd5e06193 100644 --- a/stmhal/Makefile +++ b/stmhal/Makefile @@ -51,7 +51,7 @@ INC += -I../lib/mp-readline INC += -I../lib/netutils INC += -I../lib/timeutils -CFLAGS_CORTEX_M = -mthumb -mabi=aapcs-linux -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion +CFLAGS_CORTEX_M = -mthumb -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion CFLAGS_MCU_f4 = $(CFLAGS_CORTEX_M) -mtune=cortex-m4 -mcpu=cortex-m4 -DMCU_SERIES_F4 CFLAGS_MCU_f7 = $(CFLAGS_CORTEX_M) -mtune=cortex-m7 -mcpu=cortex-m7 -DMCU_SERIES_F7 CFLAGS_MCU_l4 = $(CFLAGS_CORTEX_M) -mtune=cortex-m4 -mcpu=cortex-m4 -DMCU_SERIES_L4 @@ -64,7 +64,7 @@ CFLAGS += -Iboards/$(BOARD) CFLAGS += -DSTM32_HAL_H='' LDFLAGS = -nostdlib -L $(LD_DIR) -T $(LD_FILE) -Map=$(@:.elf=.map) --cref -LIBS = +LIBS = $(shell $(CC) $(CFLAGS) -print-libgcc-file-name) # Remove uncalled code from the final image. CFLAGS += -fdata-sections -ffunction-sections @@ -78,9 +78,6 @@ else COPT += -Os -DNDEBUG endif -# uncomment this if you want libgcc -#LIBS += $(shell $(CC) $(CFLAGS) -print-libgcc-file-name) - SRC_LIB = $(addprefix lib/,\ libc/string0.c \ libm/math.c \ From 181f7d145002731e5baec0f2c43ac57818447f8b Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 8 Feb 2017 11:14:23 +0300 Subject: [PATCH 05/92] extmod/machine_signal: Implement Signal .on() and .off() methods. Each method asserts and deasserts signal respectively. They are equivalent to .value(1) and .value(0) but conceptually simpler (and may help to avoid confusion with inverted signals, where "asserted" state means logical 0 output). --- extmod/machine_signal.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/extmod/machine_signal.c b/extmod/machine_signal.c index fb179c438c..de6c3ff320 100644 --- a/extmod/machine_signal.c +++ b/extmod/machine_signal.c @@ -92,8 +92,22 @@ STATIC mp_obj_t signal_value(size_t n_args, const mp_obj_t *args) { } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(signal_value_obj, 1, 2, signal_value); +STATIC mp_obj_t signal_on(mp_obj_t self_in) { + mp_virtual_pin_write(self_in, 1); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(signal_on_obj, signal_on); + +STATIC mp_obj_t signal_off(mp_obj_t self_in) { + mp_virtual_pin_write(self_in, 0); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(signal_off_obj, signal_off); + STATIC const mp_rom_map_elem_t signal_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_value), MP_ROM_PTR(&signal_value_obj) }, + { MP_ROM_QSTR(MP_QSTR_on), MP_ROM_PTR(&signal_on_obj) }, + { MP_ROM_QSTR(MP_QSTR_off), MP_ROM_PTR(&signal_off_obj) }, }; STATIC MP_DEFINE_CONST_DICT(signal_locals_dict, signal_locals_dict_table); From ec7dc7f8d796b5b67772d1f40863d13fb5e19be2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 9 Feb 2017 12:03:12 +1100 Subject: [PATCH 06/92] extmod/vfs: Allow to mount a block device, not just a VFS object. If the mounted object doesn't have a "mount" method then assume it's a block device and try to detect the filesystem. Since we currently only support FAT filesystems, the behaviour is to just try and create a VfsFat object automatically, using the given block device. --- extmod/vfs.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/extmod/vfs.c b/extmod/vfs.c index 1eb26acf16..c1e32e0522 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -132,11 +132,24 @@ mp_obj_t mp_vfs_mount(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args mp_uint_t mnt_len; const char *mnt_str = mp_obj_str_get_data(pos_args[1], &mnt_len); + // see if we need to auto-detect and create the filesystem + mp_obj_t vfs_obj = pos_args[0]; + mp_obj_t dest[2]; + mp_load_method_maybe(vfs_obj, MP_QSTR_mount, dest); + if (dest[0] == MP_OBJ_NULL) { + // Input object has no mount method, assume it's a block device and try to + // auto-detect the filesystem and create the corresponding VFS entity. + // (At the moment we only support FAT filesystems.) + #if MICROPY_VFS_FAT + vfs_obj = mp_fat_vfs_type.make_new(&mp_fat_vfs_type, 1, 0, &vfs_obj); + #endif + } + // create new object mp_vfs_mount_t *vfs = m_new_obj(mp_vfs_mount_t); vfs->str = mnt_str; vfs->len = mnt_len; - vfs->obj = pos_args[0]; + vfs->obj = vfs_obj; vfs->next = NULL; // call the underlying object to do any mounting operation From 18e65691661ef8e83060d0e72d66b16cc918c8b4 Mon Sep 17 00:00:00 2001 From: dmazzella Date: Tue, 3 Jan 2017 11:00:12 +0100 Subject: [PATCH 07/92] py/objtype: Implement __delattr__ and __setattr__. This patch implements support for class methods __delattr__ and __setattr__ for customising attribute access. It is controlled by the config option MICROPY_PY_DELATTR_SETATTR and is disabled by default. --- py/mpconfig.h | 6 +++ py/objtype.c | 34 +++++++++++++++ tests/basics/class_delattr_setattr.py | 63 +++++++++++++++++++++++++++ unix/mpconfigport_coverage.h | 1 + 4 files changed, 104 insertions(+) create mode 100644 tests/basics/class_delattr_setattr.py diff --git a/py/mpconfig.h b/py/mpconfig.h index afd9a0be5e..093625a461 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -635,6 +635,12 @@ typedef double mp_float_t; #define MICROPY_PY_DESCRIPTORS (0) #endif +// Whether to support class __delattr__ and __setattr__ methods +// This costs some code size and makes all del attrs and store attrs slow +#ifndef MICROPY_PY_DELATTR_SETATTR +#define MICROPY_PY_DELATTR_SETATTR (0) +#endif + // Support for async/await/async for/async with #ifndef MICROPY_PY_ASYNC_AWAIT #define MICROPY_PY_ASYNC_AWAIT (1) diff --git a/py/objtype.c b/py/objtype.c index c20b0693e5..85e10e7624 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -533,6 +533,15 @@ STATIC void mp_obj_instance_load_attr(mp_obj_t self_in, qstr attr, mp_obj_t *des // try __getattr__ if (attr != MP_QSTR___getattr__) { + #if MICROPY_PY_DELATTR_SETATTR + // If the requested attr is __setattr__/__delattr__ then don't delegate the lookup + // to __getattr__. If we followed CPython's behaviour then __setattr__/__delattr__ + // would have already been found in the "object" base class. + if (attr == MP_QSTR___setattr__ || attr == MP_QSTR___delattr__) { + return; + } + #endif + mp_obj_t dest2[3]; mp_load_method_maybe(self_in, MP_QSTR___getattr__, dest2); if (dest2[0] != MP_OBJ_NULL) { @@ -626,10 +635,35 @@ STATIC bool mp_obj_instance_store_attr(mp_obj_t self_in, qstr attr, mp_obj_t val if (value == MP_OBJ_NULL) { // delete attribute + #if MICROPY_PY_DELATTR_SETATTR + // try __delattr__ first + mp_obj_t attr_delattr_method[3]; + mp_load_method_maybe(self_in, MP_QSTR___delattr__, attr_delattr_method); + if (attr_delattr_method[0] != MP_OBJ_NULL) { + // __delattr__ exists, so call it + attr_delattr_method[2] = MP_OBJ_NEW_QSTR(attr); + mp_call_method_n_kw(1, 0, attr_delattr_method); + return true; + } + #endif + mp_map_elem_t *elem = mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_REMOVE_IF_FOUND); return elem != NULL; } else { // store attribute + #if MICROPY_PY_DELATTR_SETATTR + // try __setattr__ first + mp_obj_t attr_setattr_method[4]; + mp_load_method_maybe(self_in, MP_QSTR___setattr__, attr_setattr_method); + if (attr_setattr_method[0] != MP_OBJ_NULL) { + // __setattr__ exists, so call it + attr_setattr_method[2] = MP_OBJ_NEW_QSTR(attr); + attr_setattr_method[3] = value; + mp_call_method_n_kw(2, 0, attr_setattr_method); + return true; + } + #endif + mp_map_lookup(&self->members, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = value; return true; } diff --git a/tests/basics/class_delattr_setattr.py b/tests/basics/class_delattr_setattr.py new file mode 100644 index 0000000000..0d061aee68 --- /dev/null +++ b/tests/basics/class_delattr_setattr.py @@ -0,0 +1,63 @@ +# test __delattr__ and __setattr__ + +# feature test for __setattr__/__delattr__ +try: + class Test(): + def __delattr__(self, attr): pass + del Test().noexist +except AttributeError: + import sys + print('SKIP') + sys.exit() + +# this class just prints the calls to see if they were executed +class A(): + def __getattr__(self, attr): + print('get', attr) + return 1 + def __setattr__(self, attr, val): + print('set', attr, val) + def __delattr__(self, attr): + print('del', attr) +a = A() + +# check basic behaviour +print(getattr(a, 'foo')) +setattr(a, 'bar', 2) +delattr(a, 'baz') + +# check meta behaviour +getattr(a, '__getattr__') # should not call A.__getattr__ +getattr(a, '__setattr__') # should not call A.__getattr__ +getattr(a, '__delattr__') # should not call A.__getattr__ +setattr(a, '__setattr__', 1) # should call A.__setattr__ +delattr(a, '__delattr__') # should call A.__delattr__ + +# this class acts like a dictionary +class B: + def __init__(self, d): + # store the dict in the class, not instance, so + # we don't get infinite recursion in __getattr_ + B.d = d + + def __getattr__(self, attr): + if attr in B.d: + return B.d[attr] + else: + raise AttributeError(attr) + + def __setattr__(self, attr, value): + B.d[attr] = value + + def __delattr__(self, attr): + del B.d[attr] + +a = B({"a":1, "b":2}) +print(a.a, a.b) +a.a = 3 +print(a.a, a.b) +del a.a +try: + print(a.a) +except AttributeError: + print("AttributeError") diff --git a/unix/mpconfigport_coverage.h b/unix/mpconfigport_coverage.h index 87a743cf87..9df8d0fca8 100644 --- a/unix/mpconfigport_coverage.h +++ b/unix/mpconfigport_coverage.h @@ -32,6 +32,7 @@ #include +#define MICROPY_PY_DELATTR_SETATTR (1) #define MICROPY_PY_BUILTINS_HELP (1) #define MICROPY_PY_BUILTINS_HELP_MODULES (1) #define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) From c8febe631af902de0878c48820346fac87acecd4 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Wed, 8 Feb 2017 20:33:54 +0100 Subject: [PATCH 08/92] examples/hwapi: Add hwconfig_pyboard.py for pyboard. --- examples/hwapi/hwconfig_pyboard.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 examples/hwapi/hwconfig_pyboard.py diff --git a/examples/hwapi/hwconfig_pyboard.py b/examples/hwapi/hwconfig_pyboard.py new file mode 100644 index 0000000000..4db4324919 --- /dev/null +++ b/examples/hwapi/hwconfig_pyboard.py @@ -0,0 +1,13 @@ +from machine import Pin, Signal + +# Red LED on pin LED_RED also kown as A13 +LED = Signal(Pin('LED_RED', Pin.OUT)) + +# Green LED on pin LED_GREEN also known as A14 +LED2 = Signal(Pin('LED_GREEN', Pin.OUT)) + +# Yellow LED on pin LED_YELLOW also known as A15 +LED3 = Signal(Pin('LED_YELLOW', Pin.OUT)) + +# Blue LED on pin LED_BLUE also known as B4 +LED4 = Signal(Pin('LED_BLUE', Pin.OUT)) From c66c39313026413472039d6dc0394e1903414713 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 9 Feb 2017 15:21:57 +1100 Subject: [PATCH 09/92] examples/hwapi: Be sure to import Signal when it's used. --- examples/hwapi/hwconfig_dragonboard410c.py | 2 +- examples/hwapi/hwconfig_z_frdm_k64f.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/hwapi/hwconfig_dragonboard410c.py b/examples/hwapi/hwconfig_dragonboard410c.py index 2db21165d2..eec3582039 100644 --- a/examples/hwapi/hwconfig_dragonboard410c.py +++ b/examples/hwapi/hwconfig_dragonboard410c.py @@ -1,4 +1,4 @@ -from machine import Pin +from machine import Pin, Signal # 96Boards/Qualcomm DragonBoard 410c # diff --git a/examples/hwapi/hwconfig_z_frdm_k64f.py b/examples/hwapi/hwconfig_z_frdm_k64f.py index 583e8d7aa4..6313e2ddb5 100644 --- a/examples/hwapi/hwconfig_z_frdm_k64f.py +++ b/examples/hwapi/hwconfig_z_frdm_k64f.py @@ -1,4 +1,4 @@ -from machine import Pin +from machine import Pin, Signal # Freescale/NXP FRDM-K64F board # Blue LED on port B, pin 21 From 8f1c6d952ac695bc41afe8965aa79fbabcc6bcc1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 9 Feb 2017 15:51:34 +1100 Subject: [PATCH 10/92] extmod/vfs: Raise OSError(EEXIST) on attempt to mkdir a mount point. --- extmod/vfs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extmod/vfs.c b/extmod/vfs.c index c1e32e0522..98d8711e4a 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -275,6 +275,9 @@ MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vfs_listdir_obj, 0, 1, mp_vfs_listdir); mp_obj_t mp_vfs_mkdir(mp_obj_t path_in) { mp_obj_t path_out; mp_vfs_mount_t *vfs = lookup_path(path_in, &path_out); + if (vfs == MP_VFS_ROOT || (vfs != MP_VFS_NONE && !strcmp(mp_obj_str_get_str(path_out), "/"))) { + mp_raise_OSError(MP_EEXIST); + } return mp_vfs_proxy_call(vfs, MP_QSTR_mkdir, 1, &path_out); } MP_DEFINE_CONST_FUN_OBJ_1(mp_vfs_mkdir_obj, mp_vfs_mkdir); From cc2dbdd1fe4a5131a40b2ab73195786451314bf8 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 10 Feb 2017 11:58:10 +1100 Subject: [PATCH 11/92] py/emitbc: Produce correct line number info for large bytecode chunks. Previous to this patch, for large chunks of bytecode that originated from a single source-code line, the bytecode-line mapping would generate something like (for 42 bytecode bytes and 1 line): BC_SKIP=31 LINE_SKIP=1 BC_SKIP=11 LINE_SKIP=0 This would mean that any errors in the last 11 bytecode bytes would be reported on the following line. This patch fixes it to generate instead: BC_SKIP=31 LINE_SKIP=0 BC_SKIP=11 LINE_SKIP=1 --- py/emitbc.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/py/emitbc.c b/py/emitbc.c index e11c9ae94f..e3a047f272 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -144,10 +144,15 @@ STATIC void emit_write_code_info_bytes_lines(emit_t *emit, mp_uint_t bytes_to_sk //printf(" %d %d\n", bytes_to_skip, lines_to_skip); while (bytes_to_skip > 0 || lines_to_skip > 0) { mp_uint_t b, l; - if (lines_to_skip <= 6) { + if (lines_to_skip <= 6 || bytes_to_skip > 0xf) { // use 0b0LLBBBBB encoding b = MIN(bytes_to_skip, 0x1f); - l = MIN(lines_to_skip, 0x3); + if (b < bytes_to_skip) { + // we can't skip any lines until we skip all the bytes + l = 0; + } else { + l = MIN(lines_to_skip, 0x3); + } *emit_get_cur_to_write_code_info(emit, 1) = b | (l << 5); } else { // use 0b1LLLBBBB 0bLLLLLLLL encoding (l's LSB in second byte) From e08395a35c37fa7f7c0311cc4c7a71537b8b4227 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 10 Feb 2017 12:02:02 +1100 Subject: [PATCH 12/92] tests/misc: Add test for line number printing with large bytecode chunk. --- tests/misc/print_exception.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/misc/print_exception.py b/tests/misc/print_exception.py index 9baac713ee..30a732106c 100644 --- a/tests/misc/print_exception.py +++ b/tests/misc/print_exception.py @@ -41,3 +41,13 @@ try: except Exception as e: print('caught') print_exc(e) + +# Here we have a function with lots of bytecode generated for a single source-line, and +# there is an error right at the end of the bytecode. It should report the correct line. +def f(): + f([1, 2], [1, 2], [1, 2], {1:1, 1:1, 1:1, 1:1, 1:1, 1:1, 1:X}) + return 1 +try: + f() +except Exception as e: + print_exc(e) From aac2db9aafbaaa7cea82be5683b69833859200f5 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 10 Feb 2017 20:18:05 +0300 Subject: [PATCH 13/92] tools/upip: Update to 1.1.5. Better and more user-friendly error handling. --- tools/upip.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/upip.py b/tools/upip.py index db18a7427e..0070fd6195 100644 --- a/tools/upip.py +++ b/tools/upip.py @@ -105,7 +105,10 @@ warn_ussl = True def url_open(url): global warn_ussl proto, _, host, urlpath = url.split('/', 3) - ai = usocket.getaddrinfo(host, 443) + try: + ai = usocket.getaddrinfo(host, 443) + except OSError as e: + fatal("Unable to resolve %s (no Internet?)" % host, e) #print("Address infos:", ai) addr = ai[0][4] @@ -124,13 +127,16 @@ def url_open(url): l = s.readline() protover, status, msg = l.split(None, 2) if status != b"200": + s.close() + exc = ValueError(status) if status == b"404": - print("Package not found") - raise ValueError(status) + fatal("Package not found", exc) + fatal("Unexpected error querying for package", exc) while 1: l = s.readline() if not l: - raise ValueError("Unexpected EOF") + s.close() + fatal("Unexpected EOF in HTTP headers", ValueError()) if l == b'\r\n': break @@ -144,8 +150,10 @@ def get_pkg_metadata(name): return json.loads(s) -def fatal(msg): - print(msg) +def fatal(msg, exc=None): + print("Error:", msg) + if exc and debug: + raise exc sys.exit(1) def install_pkg(pkg_spec, install_path): From 16a3534ad44e7de1b8c54bbe1f5a01d3636db780 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Mon, 13 Feb 2017 00:23:23 +0300 Subject: [PATCH 14/92] unix/moduselect: Implement ipoll() method with no-allocation policy. ipoll() allows to poll streams without allocating any memory: this method returns an iterator (a poll object itself), and the iterator yields preallocated "callee-owned tuple" with polling results for each active stream. The only operation a caller is allowed to do with this tuple is extracting values from it (storing the tuple as a whole somewhere is not allowed). --- unix/moduselect.c | 77 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/unix/moduselect.c b/unix/moduselect.c index e74c44be66..f966efa412 100644 --- a/unix/moduselect.c +++ b/unix/moduselect.c @@ -4,7 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2014 Damien P. George - * Copyright (c) 2015 Paul Sokolovsky + * Copyright (c) 2015-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -55,6 +55,11 @@ typedef struct _mp_obj_poll_t { unsigned short len; struct pollfd *entries; mp_obj_t *obj_map; + short iter_cnt; + short iter_idx; + int flags; + // callee-owned tuple + mp_obj_t ret_tuple; } mp_obj_poll_t; STATIC int get_fd(mp_obj_t fdlike) { @@ -164,9 +169,7 @@ STATIC mp_obj_t poll_modify(mp_obj_t self_in, mp_obj_t obj_in, mp_obj_t eventmas } MP_DEFINE_CONST_FUN_OBJ_3(poll_modify_obj, poll_modify); -/// \method poll([timeout]) -/// Timeout is in milliseconds. -STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { +STATIC int poll_poll_internal(size_t n_args, const mp_obj_t *args) { mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); // work out timeout (it's given already in ms) @@ -184,12 +187,24 @@ STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { } } + self->flags = flags; + int n_ready = poll(self->entries, self->len, timeout); RAISE_ERRNO(n_ready, errno); + return n_ready; +} + +/// \method poll([timeout]) +/// Timeout is in milliseconds. +STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { + int n_ready = poll_poll_internal(n_args, args); + if (n_ready == 0) { return mp_const_empty_tuple; } + mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); + mp_obj_list_t *ret_list = MP_OBJ_TO_PTR(mp_obj_new_list(n_ready, NULL)); int ret_i = 0; struct pollfd *entries = self->entries; @@ -204,7 +219,7 @@ STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { } t->items[1] = MP_OBJ_NEW_SMALL_INT(entries->revents); ret_list->items[ret_i++] = MP_OBJ_FROM_PTR(t); - if (flags & FLAG_ONESHOT) { + if (self->flags & FLAG_ONESHOT) { entries->events = 0; } } @@ -214,17 +229,67 @@ STATIC mp_obj_t poll_poll(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_poll_obj, 1, 3, poll_poll); +STATIC mp_obj_t poll_ipoll(size_t n_args, const mp_obj_t *args) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(args[0]); + + if (self->ret_tuple == MP_OBJ_NULL) { + self->ret_tuple = mp_obj_new_tuple(2, NULL); + } + + int n_ready = poll_poll_internal(n_args, args); + self->iter_cnt = n_ready; + self->iter_idx = 0; + + return args[0]; +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(poll_ipoll_obj, 1, 3, poll_ipoll); + +STATIC mp_obj_t poll_iternext(mp_obj_t self_in) { + mp_obj_poll_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->iter_cnt == 0) { + return MP_OBJ_STOP_ITERATION; + } + + self->iter_cnt--; + + struct pollfd *entries = self->entries; + for (int i = self->iter_idx; i < self->len; i++, entries++) { + if (entries->revents != 0) { + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(self->ret_tuple); + // If there's an object stored, return it, otherwise raw fd + if (self->obj_map && self->obj_map[i] != MP_OBJ_NULL) { + t->items[0] = self->obj_map[i]; + } else { + t->items[0] = MP_OBJ_NEW_SMALL_INT(entries->fd); + } + t->items[1] = MP_OBJ_NEW_SMALL_INT(entries->revents); + if (self->flags & FLAG_ONESHOT) { + entries->events = 0; + } + return MP_OBJ_FROM_PTR(t); + } + } + + assert(!"inconsistent number of poll active entries"); + self->iter_cnt = 0; + return MP_OBJ_STOP_ITERATION; +} + STATIC const mp_rom_map_elem_t poll_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_register), MP_ROM_PTR(&poll_register_obj) }, { MP_ROM_QSTR(MP_QSTR_unregister), MP_ROM_PTR(&poll_unregister_obj) }, { MP_ROM_QSTR(MP_QSTR_modify), MP_ROM_PTR(&poll_modify_obj) }, { MP_ROM_QSTR(MP_QSTR_poll), MP_ROM_PTR(&poll_poll_obj) }, + { MP_ROM_QSTR(MP_QSTR_ipoll), MP_ROM_PTR(&poll_ipoll_obj) }, }; STATIC MP_DEFINE_CONST_DICT(poll_locals_dict, poll_locals_dict_table); STATIC const mp_obj_type_t mp_type_poll = { { &mp_type_type }, .name = MP_QSTR_poll, + .getiter = mp_identity, + .iternext = poll_iternext, .locals_dict = (void*)&poll_locals_dict, }; @@ -239,6 +304,8 @@ STATIC mp_obj_t select_poll(size_t n_args, const mp_obj_t *args) { poll->alloc = alloc; poll->len = 0; poll->obj_map = NULL; + poll->iter_cnt = 0; + poll->ret_tuple = MP_OBJ_NULL; return MP_OBJ_FROM_PTR(poll); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_select_poll_obj, 0, 1, select_poll); From f92f7dd2bc51848fb08651f91b09ef66b8b96817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20W=C3=A5gen?= Date: Tue, 7 Feb 2017 21:46:41 +0100 Subject: [PATCH 15/92] stmhal/boards: For STM32F411DISC, change I2C pin according to datasheet. The pin had to be changed to get the I2C sensors on board to work. --- stmhal/boards/STM32F411DISC/mpconfigboard.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stmhal/boards/STM32F411DISC/mpconfigboard.h b/stmhal/boards/STM32F411DISC/mpconfigboard.h index d87701f8af..1488cd7640 100644 --- a/stmhal/boards/STM32F411DISC/mpconfigboard.h +++ b/stmhal/boards/STM32F411DISC/mpconfigboard.h @@ -35,7 +35,7 @@ // I2C busses #define MICROPY_HW_I2C1_SCL (pin_B6) -#define MICROPY_HW_I2C1_SDA (pin_B7) +#define MICROPY_HW_I2C1_SDA (pin_B9) //#define MICROPY_HW_I2C2_SCL (pin_B10) //#define MICROPY_HW_I2C2_SDA (pin_B11) #define MICROPY_HW_I2C3_SCL (pin_A8) From 39100dc3777382de75a63bb5edd36ea73d41d776 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 13 Feb 2017 11:47:17 +1100 Subject: [PATCH 16/92] esp8266/moduos: Populate release field of uname in case it was GC'd. --- esp8266/moduos.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esp8266/moduos.c b/esp8266/moduos.c index 006fc49d61..a22fbd4df0 100644 --- a/esp8266/moduos.c +++ b/esp8266/moduos.c @@ -58,10 +58,9 @@ STATIC mp_obj_tuple_t os_uname_info_obj = { }; STATIC mp_obj_t os_uname(void) { - if (os_uname_info_obj.items[2] == NULL) { - const char *ver = system_get_sdk_version(); - os_uname_info_obj.items[2] = mp_obj_new_str(ver, strlen(ver), false); - } + // We must populate the "release" field each time in case it was GC'd since the last call. + const char *ver = system_get_sdk_version(); + os_uname_info_obj.items[2] = mp_obj_new_str(ver, strlen(ver), false); return (mp_obj_t)&os_uname_info_obj; } STATIC MP_DEFINE_CONST_FUN_OBJ_0(os_uname_obj, os_uname); From 2f76c3ca0ad809e26f12722192e6fbdeab317566 Mon Sep 17 00:00:00 2001 From: Dave Hylands Date: Sat, 11 Feb 2017 09:39:39 -0800 Subject: [PATCH 17/92] docs/library/pyb.Pin: Minor typo fix, B6 should be A0. On the PYBv1.0, X1 maps to A0, not B6. --- docs/library/pyb.Pin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/library/pyb.Pin.rst b/docs/library/pyb.Pin.rst index 4f589fff89..b766c5280c 100644 --- a/docs/library/pyb.Pin.rst +++ b/docs/library/pyb.Pin.rst @@ -21,7 +21,7 @@ Usage Model: CPU pins which correspond to the board pins are available as ``pyb.cpu.Name``. For the CPU pins, the names are the port letter followed by the pin number. On the PYBv1.0, ``pyb.Pin.board.X1`` and - ``pyb.Pin.cpu.B6`` are the same pin. + ``pyb.Pin.cpu.A0`` are the same pin. You can also use strings:: From 3625afa17362e8a2f873edeaa0856ae852c19beb Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 13 Feb 2017 12:25:43 +1100 Subject: [PATCH 18/92] extmod/vfs: Allow to stat the root directory. os.stat('/') now works and returns a mostly-empty tuple. Really all that is useful is the mode which tells that it's a directory. --- extmod/vfs.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/extmod/vfs.c b/extmod/vfs.c index 98d8711e4a..c968345b86 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -311,6 +311,14 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_vfs_rmdir_obj, mp_vfs_rmdir); mp_obj_t mp_vfs_stat(mp_obj_t path_in) { mp_obj_t path_out; mp_vfs_mount_t *vfs = lookup_path(path_in, &path_out); + if (vfs == MP_VFS_ROOT) { + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(0x4000); // st_mode = stat.S_IFDIR + for (int i = 1; i <= 9; ++i) { + t->items[i] = MP_OBJ_NEW_SMALL_INT(0); // dev, nlink, uid, gid, size, atime, mtime, ctime + } + return MP_OBJ_FROM_PTR(t); + } return mp_vfs_proxy_call(vfs, MP_QSTR_stat, 1, &path_out); } MP_DEFINE_CONST_FUN_OBJ_1(mp_vfs_stat_obj, mp_vfs_stat); From 0c821f7def92a2f461a440c210cd5944707d4244 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 13 Feb 2017 13:06:51 +1100 Subject: [PATCH 19/92] docs/library/machine: Make separate TOC for WiPy vs non-WiPy. WiPy is the only port with ADC and SD, so they shouldn't be included in other ports' documentation. --- docs/library/machine.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/library/machine.rst b/docs/library/machine.rst index c6da715852..50884e7be0 100644 --- a/docs/library/machine.rst +++ b/docs/library/machine.rst @@ -156,7 +156,22 @@ Constants Classes ------- -.. toctree:: +.. only:: not port_wipy + + .. toctree:: + :maxdepth: 1 + + machine.I2C.rst + machine.Pin.rst + machine.RTC.rst + machine.SPI.rst + machine.Timer.rst + machine.UART.rst + machine.WDT.rst + +.. only:: port_wipy + + .. toctree:: :maxdepth: 1 machine.ADC.rst From adc80b8f8469874624a200acaddbed901bdd34ee Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 14 Feb 2017 20:55:31 +1100 Subject: [PATCH 20/92] py/objtype: Replace non-ASCII single-quote char with ASCII version. --- py/objtype.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/objtype.c b/py/objtype.c index 85e10e7624..60f630c3dc 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -284,7 +284,7 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size } // https://docs.python.org/3.4/reference/datamodel.html#object.__new__ - // "If __new__() does not return an instance of cls, then the new instance’s __init__() method will not be invoked." + // "If __new__() does not return an instance of cls, then the new instance's __init__() method will not be invoked." if (mp_obj_get_type(new_ret) != self) { return new_ret; } From dd00d0134bf4a15fe4f72dc8a157587052717c59 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 13:11:49 +0300 Subject: [PATCH 21/92] examples/hwapi/soft_pwm: Use Signal on()/off() methods. Just one sample is updated with on()/off() for now, there should be remaining sample(s) showing .value() use (but more can be converted later, as long as 1 or so good samples of .value() remains). --- examples/hwapi/soft_pwm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/hwapi/soft_pwm.py b/examples/hwapi/soft_pwm.py index a9a5561717..72291b0ecd 100644 --- a/examples/hwapi/soft_pwm.py +++ b/examples/hwapi/soft_pwm.py @@ -14,10 +14,10 @@ def pwm_cycle(led, duty, cycles): duty_off = 20 - duty for i in range(cycles): if duty: - led.value(1) + led.on() utime.sleep_ms(duty) if duty_off: - led.value(0) + led.off() utime.sleep_ms(duty_off) From ee3615d80022d1b5e5c0fdca466f20e50f07e63c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 13:14:46 +0300 Subject: [PATCH 22/92] docs/uos: Remove mention of uos.sep. MicroPython guarantees '/' to be a path separator, so extra constant taking precious ROM space are not needed. MicroPython never had such constant, only one vendor port had it (now unmaintained). --- docs/library/uos.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/library/uos.rst b/docs/library/uos.rst index 242e8c3d0b..cd0c5cae8c 100644 --- a/docs/library/uos.rst +++ b/docs/library/uos.rst @@ -116,10 +116,3 @@ Functions Duplicate the terminal (the REPL) on the passed stream-like object. The given object must at least implement the ``.read()`` and ``.write()`` methods. - -Constants ---------- - -.. data:: sep - - separation character used in paths From 64916436b2b6ded6770cd58d3cb4f5382c2f7d1e Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 16:23:54 +0300 Subject: [PATCH 23/92] zephyr: Enable IPv6 networking in addition to IPv4. --- zephyr/main.c | 9 +++++++-- zephyr/prj_base.conf | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/zephyr/main.c b/zephyr/main.c index 6d7fca8024..a11a6dbdab 100644 --- a/zephyr/main.c +++ b/zephyr/main.c @@ -67,11 +67,16 @@ static char *stack_top; static char heap[MICROPY_HEAP_SIZE]; void init_zephyr(void) { + // TODO: Make addresses configurable #ifdef CONFIG_NET_IPV4 - // TODO: Make address configurable - static struct in_addr in4addr_my = { { { 192, 0, 2, 1 } } }; + static struct in_addr in4addr_my = {{{192, 0, 2, 1}}}; net_if_ipv4_addr_add(net_if_get_default(), &in4addr_my, NET_ADDR_MANUAL, 0); #endif + #ifdef CONFIG_NET_IPV6 + // 2001:db8::1 + static struct in6_addr in6addr_my = {{{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}}; + net_if_ipv6_addr_add(net_if_get_default(), &in6addr_my, NET_ADDR_MANUAL, 0); + #endif } int real_main(void) { diff --git a/zephyr/prj_base.conf b/zephyr/prj_base.conf index 43aedd8fec..295963023b 100644 --- a/zephyr/prj_base.conf +++ b/zephyr/prj_base.conf @@ -8,5 +8,6 @@ CONFIG_MAIN_STACK_SIZE=4096 # Networking config CONFIG_NETWORKING=y CONFIG_NET_IPV4=y +CONFIG_NET_IPV6=y CONFIG_TEST_RANDOM_GENERATOR=y -CONFIG_NET_NBUF_RX_COUNT=4 +CONFIG_NET_NBUF_RX_COUNT=5 From ce2703599f844b73f3eab10b96197a01bc01ccf2 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 17:01:26 +0300 Subject: [PATCH 24/92] zephyr: Add qemu_cortex_m3 config fragment. Should work for QEMU networking with soon-to-merged upstream patch. --- zephyr/prj_qemu_cortex_m3.conf | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 zephyr/prj_qemu_cortex_m3.conf diff --git a/zephyr/prj_qemu_cortex_m3.conf b/zephyr/prj_qemu_cortex_m3.conf new file mode 100644 index 0000000000..09614c3620 --- /dev/null +++ b/zephyr/prj_qemu_cortex_m3.conf @@ -0,0 +1,3 @@ +# Networking drivers +# SLIP driver for QEMU +CONFIG_NET_SLIP_TAP=y From c6fd9ba4f371dc8f2ec5e1859587fafa15bef9d0 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 24 Mar 2015 11:55:26 +0100 Subject: [PATCH 25/92] tests: Add option to not clear MICROPYPATH when running tests This allows using the test runner for other scenarios than just testing uPy itself. The principle of comparing either to CPython or else to a .exp file is really handy but to be able to test custom modules not built into micropython.exe one needs to be able to specify the module search path a.k.a MICROPYPATH. --- tests/run-tests | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/run-tests b/tests/run-tests index c74d0f24b4..24599ed64e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -23,7 +23,6 @@ MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross') # Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale os.environ['PYTHONIOENCODING'] = 'utf-8' -os.environ['MICROPYPATH'] = '' def rm_f(fname): if os.path.exists(fname): @@ -400,6 +399,7 @@ def main(): cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)') cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)') cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first') + cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests') cmd_parser.add_argument('files', nargs='*', help='input test files') args = cmd_parser.parse_args() @@ -434,6 +434,10 @@ def main(): # tests explicitly given tests = args.files + if not args.keep_path: + # clear search path to make sure tests use only builtin modules + os.environ['MICROPYPATH'] = '' + if not run_tests(pyb, tests, args): sys.exit(1) From ce2e0eeb7b0eeb8d5f188ec98e1f1b3d1307ca9c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 22:03:25 +0300 Subject: [PATCH 26/92] tests/run-tests: Allow to skip tests using async/await keywords. --- tests/feature_check/async_check.py | 3 +++ tests/feature_check/async_check.py.exp | 0 tests/run-tests | 8 ++++++++ 3 files changed, 11 insertions(+) create mode 100644 tests/feature_check/async_check.py create mode 100644 tests/feature_check/async_check.py.exp diff --git a/tests/feature_check/async_check.py b/tests/feature_check/async_check.py new file mode 100644 index 0000000000..0f6361cd12 --- /dev/null +++ b/tests/feature_check/async_check.py @@ -0,0 +1,3 @@ +# check if async/await keywords are supported +async def foo(): + await 1 diff --git a/tests/feature_check/async_check.py.exp b/tests/feature_check/async_check.py.exp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/run-tests b/tests/run-tests index 24599ed64e..4bebe8d47f 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -199,6 +199,7 @@ def run_tests(pyb, tests, args): skip_tests = set() skip_native = False skip_set_type = False + skip_async = False # Check if micropython.native is supported, and skip such tests if it's not native = run_micropython(pyb, args, 'feature_check/native_check.py') @@ -210,6 +211,11 @@ def run_tests(pyb, tests, args): if native == b'CRASH': skip_set_type = True + # Check if async/await keywords are supported, and skip such tests if it's not + native = run_micropython(pyb, args, 'feature_check/async_check.py') + if native == b'CRASH': + skip_async = True + # Check if emacs repl is supported, and skip such tests if it's not t = run_micropython(pyb, args, 'feature_check/repl_emacs_check.py') if not 'True' in str(t, 'ascii'): @@ -314,11 +320,13 @@ def run_tests(pyb, tests, args): is_native = test_name.startswith("native_") or test_name.startswith("viper_") is_endian = test_name.endswith("_endian") is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset") + is_async = test_name.startswith("async_") skip_it = test_file in skip_tests skip_it |= skip_native and is_native skip_it |= skip_endian and is_endian skip_it |= skip_set_type and is_set_type + skip_it |= skip_async and is_async if skip_it: print("skip ", test_file) From 800b163cd8713df8a2b1c87f23c40eb80290d918 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 22:22:45 +0300 Subject: [PATCH 27/92] tests/comprehension1, containment: Split set tests to separate files. To make skippable. --- tests/basics/comprehension1.py | 3 +-- tests/basics/containment.py | 3 ++- tests/basics/set_comprehension.py | 1 + tests/basics/set_containment.py | 4 ++++ 4 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 tests/basics/set_comprehension.py create mode 100644 tests/basics/set_containment.py diff --git a/tests/basics/comprehension1.py b/tests/basics/comprehension1.py index 7f541ee53f..892d6b4e33 100644 --- a/tests/basics/comprehension1.py +++ b/tests/basics/comprehension1.py @@ -14,7 +14,6 @@ def f(): print(d[0], d[1], d[2], d[3], d[4]) # set comprehension - - print({a for a in range(5)}) + # see set_comprehension.py f() diff --git a/tests/basics/containment.py b/tests/basics/containment.py index f8be04e929..bae3661131 100644 --- a/tests/basics/containment.py +++ b/tests/basics/containment.py @@ -1,5 +1,6 @@ +# sets, see set_containment for i in 1, 2: - for o in {1:2}, {1}, {1:2}.keys(): + for o in {1:2}, {1:2}.keys(): print("{} in {}: {}".format(i, o, i in o)) print("{} not in {}: {}".format(i, o, i not in o)) diff --git a/tests/basics/set_comprehension.py b/tests/basics/set_comprehension.py new file mode 100644 index 0000000000..12a9a29d31 --- /dev/null +++ b/tests/basics/set_comprehension.py @@ -0,0 +1 @@ +print({a for a in range(5)}) diff --git a/tests/basics/set_containment.py b/tests/basics/set_containment.py new file mode 100644 index 0000000000..97694f74c5 --- /dev/null +++ b/tests/basics/set_containment.py @@ -0,0 +1,4 @@ +for i in 1, 2: + for o in {}, {1}, {2}: + print("{} in {}: {}".format(i, o, i in o)) + print("{} not in {}: {}".format(i, o, i not in o)) From d61ce3202278e9ad251bde0e6075a7d841b7639e Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 23:27:44 +0300 Subject: [PATCH 28/92] tests/builtin_dir: The most expected thing in sys is exit, test for it. --- tests/basics/builtin_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basics/builtin_dir.py b/tests/basics/builtin_dir.py index 843467e783..16e7e669e4 100644 --- a/tests/basics/builtin_dir.py +++ b/tests/basics/builtin_dir.py @@ -5,7 +5,7 @@ print('__name__' in dir()) # dir of module import sys -print('platform' in dir(sys)) +print('exit' in dir(sys)) # dir of type print('append' in dir(list)) From 453f98914e66b65f7335fb6d24269130c51777be Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Tue, 14 Feb 2017 23:30:55 +0300 Subject: [PATCH 29/92] zephyr/main: Don't unconditionally dump stats on each GC. This was a debug output for initial porting, breaks tests. --- zephyr/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zephyr/main.c b/zephyr/main.c index a11a6dbdab..d6980ad295 100644 --- a/zephyr/main.c +++ b/zephyr/main.c @@ -126,7 +126,7 @@ void gc_collect(void) { gc_collect_start(); gc_collect_root(&dummy, ((mp_uint_t)stack_top - (mp_uint_t)&dummy) / sizeof(mp_uint_t)); gc_collect_end(); - gc_dump_info(); + //gc_dump_info(); } mp_lexer_t *mp_lexer_new_from_file(const char *filename) { From 83623b2fdeee0ada918ee4f91a99f3d732c3dee7 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 00:57:56 +0300 Subject: [PATCH 30/92] tests/basic/[a-f]*: Make skippable. For small ports which don't have all features enabled. --- tests/basics/array_micropython.py | 7 ++++++- tests/basics/attrtuple1.py | 9 +++++++++ tests/basics/builtin_delattr.py | 6 ++++++ tests/basics/builtin_minmax.py | 7 +++++++ tests/basics/builtin_override.py | 8 +++++++- tests/basics/builtin_range.py | 11 ----------- tests/basics/builtin_range_attrs.py | 19 +++++++++++++++++++ tests/basics/builtin_reversed.py | 6 ++++++ tests/basics/class_descriptor.py | 7 +++++++ tests/basics/class_new.py | 8 ++++++++ tests/basics/class_store_class.py | 7 ++++++- tests/basics/class_super_object.py | 8 ++++++++ tests/basics/dict_fromkeys.py | 3 ++- tests/basics/enumerate.py | 7 +++++++ tests/basics/filter.py | 7 +++++++ 15 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 tests/basics/builtin_range_attrs.py diff --git a/tests/basics/array_micropython.py b/tests/basics/array_micropython.py index 8e904bdfea..0c1df0923b 100644 --- a/tests/basics/array_micropython.py +++ b/tests/basics/array_micropython.py @@ -1,5 +1,10 @@ # test MicroPython-specific features of array.array -import array +try: + import array +except ImportError: + import sys + print("SKIP") + sys.exit() # arrays of objects a = array.array('O') diff --git a/tests/basics/attrtuple1.py b/tests/basics/attrtuple1.py index c4daaaf257..597bfc2a32 100644 --- a/tests/basics/attrtuple1.py +++ b/tests/basics/attrtuple1.py @@ -4,6 +4,15 @@ import sys t = sys.implementation +# It can be just a normal tuple on small ports +try: + t.name +except AttributeError: + import sys + print("SKIP") + sys.exit() + + # test printing of attrtuple print(str(t).find("version=") > 0) diff --git a/tests/basics/builtin_delattr.py b/tests/basics/builtin_delattr.py index 3743df227c..9b38837e44 100644 --- a/tests/basics/builtin_delattr.py +++ b/tests/basics/builtin_delattr.py @@ -1,4 +1,10 @@ # test builtin delattr +try: + delattr +except: + import sys + print("SKIP") + sys.exit() class A: pass a = A() diff --git a/tests/basics/builtin_minmax.py b/tests/basics/builtin_minmax.py index d395d4421b..a925b3fe92 100644 --- a/tests/basics/builtin_minmax.py +++ b/tests/basics/builtin_minmax.py @@ -1,4 +1,11 @@ # test builtin min and max functions +try: + min + max +except: + import sys + print("SKIP") + sys.exit() print(min(0,1)) print(min(1,0)) diff --git a/tests/basics/builtin_override.py b/tests/basics/builtin_override.py index e245985ad9..f3632e59a7 100644 --- a/tests/basics/builtin_override.py +++ b/tests/basics/builtin_override.py @@ -3,7 +3,13 @@ import builtins # override generic builtin -builtins.abs = lambda x: x + 1 +try: + builtins.abs = lambda x: x + 1 +except AttributeError: + import sys + print("SKIP") + sys.exit() + print(abs(1)) # __build_class__ is handled in a special way diff --git a/tests/basics/builtin_range.py b/tests/basics/builtin_range.py index 59fc0344a4..7c3e5beeff 100644 --- a/tests/basics/builtin_range.py +++ b/tests/basics/builtin_range.py @@ -34,11 +34,6 @@ print(range(1, 4)[1:]) print(range(1, 4)[:-1]) print(range(7, -2, -4)[:]) -# attrs -print(range(1, 2, 3).start) -print(range(1, 2, 3).stop) -print(range(1, 2, 3).step) - # bad unary op try: -range(1) @@ -50,9 +45,3 @@ try: range(1)[0] = 1 except TypeError: print("TypeError") - -# bad attr (can't store) -try: - range(4).start = 0 -except AttributeError: - print('AttributeError') diff --git a/tests/basics/builtin_range_attrs.py b/tests/basics/builtin_range_attrs.py new file mode 100644 index 0000000000..9327c802a5 --- /dev/null +++ b/tests/basics/builtin_range_attrs.py @@ -0,0 +1,19 @@ +# test attributes of builtin range type + +try: + range(0).start +except AttributeError: + import sys + print("SKIP") + sys.exit() + +# attrs +print(range(1, 2, 3).start) +print(range(1, 2, 3).stop) +print(range(1, 2, 3).step) + +# bad attr (can't store) +try: + range(4).start = 0 +except AttributeError: + print('AttributeError') diff --git a/tests/basics/builtin_reversed.py b/tests/basics/builtin_reversed.py index f129a4f5d5..59e9c78219 100644 --- a/tests/basics/builtin_reversed.py +++ b/tests/basics/builtin_reversed.py @@ -1,4 +1,10 @@ # test the builtin reverse() function +try: + reversed +except: + import sys + print("SKIP") + sys.exit() # list print(list(reversed([]))) diff --git a/tests/basics/class_descriptor.py b/tests/basics/class_descriptor.py index 25b373e47e..7f295f071e 100644 --- a/tests/basics/class_descriptor.py +++ b/tests/basics/class_descriptor.py @@ -18,6 +18,13 @@ class Main: Forward = Descriptor() m = Main() +try: + m.__class__ +except AttributeError: + import sys + print("SKIP") + sys.exit() + r = m.Forward if 'Descriptor' in repr(r.__class__): print('SKIP') diff --git a/tests/basics/class_new.py b/tests/basics/class_new.py index a6a34c5811..0198456b24 100644 --- a/tests/basics/class_new.py +++ b/tests/basics/class_new.py @@ -1,3 +1,11 @@ +try: + # If we don't expose object.__new__ (small ports), there's + # nothing to test. + object.__new__ +except AttributeError: + import sys + print("SKIP") + sys.exit() class A: def __new__(cls): print("A.__new__") diff --git a/tests/basics/class_store_class.py b/tests/basics/class_store_class.py index 10b94d3c6a..00a2915869 100644 --- a/tests/basics/class_store_class.py +++ b/tests/basics/class_store_class.py @@ -5,7 +5,12 @@ try: from collections import namedtuple except ImportError: - from ucollections import namedtuple + try: + from ucollections import namedtuple + except ImportError: + import sys + print("SKIP") + sys.exit() _DefragResultBase = namedtuple('DefragResult', [ 'foo', 'bar' ]) diff --git a/tests/basics/class_super_object.py b/tests/basics/class_super_object.py index 21b97328ea..a841d34abb 100644 --- a/tests/basics/class_super_object.py +++ b/tests/basics/class_super_object.py @@ -1,4 +1,12 @@ # Calling object.__init__() via super().__init__ +try: + # If we don't expose object.__init__ (small ports), there's + # nothing to test. + object.__init__ +except AttributeError: + import sys + print("SKIP") + sys.exit() class Test(object): def __init__(self): diff --git a/tests/basics/dict_fromkeys.py b/tests/basics/dict_fromkeys.py index bfad347c89..796f657d7e 100644 --- a/tests/basics/dict_fromkeys.py +++ b/tests/basics/dict_fromkeys.py @@ -9,5 +9,6 @@ l.sort() print(l) # argument to fromkeys has no __len__ -d = dict.fromkeys(reversed(range(1))) +#d = dict.fromkeys(reversed(range(1))) +d = dict.fromkeys((x for x in range(1))) print(d) diff --git a/tests/basics/enumerate.py b/tests/basics/enumerate.py index 00595cb0f6..3cc1350a0f 100644 --- a/tests/basics/enumerate.py +++ b/tests/basics/enumerate.py @@ -1,3 +1,10 @@ +try: + enumerate +except: + import sys + print("SKIP") + sys.exit() + print(list(enumerate([]))) print(list(enumerate([1, 2, 3]))) print(list(enumerate([1, 2, 3], 5))) diff --git a/tests/basics/filter.py b/tests/basics/filter.py index 5883e3d00b..d0b36733c3 100644 --- a/tests/basics/filter.py +++ b/tests/basics/filter.py @@ -1,2 +1,9 @@ +try: + filter +except: + import sys + print("SKIP") + sys.exit() + print(list(filter(lambda x: x & 1, range(-3, 4)))) print(list(filter(None, range(-3, 4)))) From 7bb146350e5de1f406dc69679ecdf4cdded5be75 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 01:30:16 +0300 Subject: [PATCH 31/92] tests/dict_fromkeys: Revert to use reversed() to run in native codegen mode. --- tests/basics/dict_fromkeys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basics/dict_fromkeys.py b/tests/basics/dict_fromkeys.py index 796f657d7e..118d0ffd95 100644 --- a/tests/basics/dict_fromkeys.py +++ b/tests/basics/dict_fromkeys.py @@ -9,6 +9,6 @@ l.sort() print(l) # argument to fromkeys has no __len__ -#d = dict.fromkeys(reversed(range(1))) -d = dict.fromkeys((x for x in range(1))) +d = dict.fromkeys(reversed(range(1))) +#d = dict.fromkeys((x for x in range(1))) print(d) From f2d732f4596064b3257abe571dc14ab61e02dec9 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 01:56:22 +0300 Subject: [PATCH 32/92] tests/extmod: Make tests skippable. --- tests/extmod/machine1.py | 3 ++- tests/extmod/ubinascii_a2b_base64.py | 9 +++++++-- tests/extmod/ubinascii_b2a_base64.py | 9 +++++++-- tests/extmod/ubinascii_crc32.py | 10 ++++++++-- tests/extmod/ubinascii_hexlify.py | 9 +++++++-- tests/extmod/ubinascii_micropython.py | 9 +++++++-- tests/extmod/ubinascii_unhexlify.py | 9 +++++++-- tests/extmod/uctypes_array_assign_le.py | 7 ++++++- tests/extmod/uctypes_array_assign_native_le.py | 6 +++++- tests/extmod/uctypes_bytearray.py | 7 ++++++- tests/extmod/uctypes_le.py | 7 ++++++- tests/extmod/uctypes_le_float.py | 7 ++++++- tests/extmod/uctypes_native_float.py | 7 ++++++- tests/extmod/uctypes_native_le.py | 6 +++++- tests/extmod/uctypes_print.py | 8 ++++++-- tests/extmod/uctypes_ptr_le.py | 6 +++++- tests/extmod/uctypes_ptr_native_le.py | 6 +++++- tests/extmod/uctypes_sizeof.py | 7 ++++++- tests/extmod/uctypes_sizeof_native.py | 7 ++++++- tests/extmod/uheapq1.py | 7 ++++++- tests/extmod/ujson_dumps.py | 7 ++++++- tests/extmod/ujson_dumps_extra.py | 7 ++++++- tests/extmod/ujson_dumps_float.py | 7 ++++++- tests/extmod/ujson_load.py | 9 +++++++-- tests/extmod/ujson_loads.py | 9 +++++++-- tests/extmod/ujson_loads_float.py | 9 +++++++-- tests/extmod/urandom_basic.py | 7 ++++++- tests/extmod/urandom_extra.py | 7 ++++++- tests/extmod/ure1.py | 7 ++++++- tests/extmod/ure_debug.py | 8 +++++++- tests/extmod/ure_error.py | 9 +++++++-- tests/extmod/ure_group.py | 9 +++++++-- tests/extmod/ure_namedclass.py | 9 +++++++-- tests/extmod/ure_split.py | 7 ++++++- tests/extmod/ure_split_empty.py | 7 ++++++- tests/extmod/ure_split_notimpl.py | 7 ++++++- tests/extmod/uzlib_decompio.py | 8 +++++--- tests/extmod/uzlib_decompio_gz.py | 8 +++++--- tests/extmod/uzlib_decompress.py | 7 ++++++- tests/extmod/vfs_fat_oldproto.py | 11 ++++++++--- tests/extmod/vfs_fat_ramdisk.py | 11 ++++++++--- 41 files changed, 254 insertions(+), 62 deletions(-) diff --git a/tests/extmod/machine1.py b/tests/extmod/machine1.py index 433a180376..e0c5611684 100644 --- a/tests/extmod/machine1.py +++ b/tests/extmod/machine1.py @@ -5,7 +5,8 @@ try: import umachine as machine except ImportError: import machine -except ImportError: + machine.mem8 +except: print("SKIP") import sys sys.exit() diff --git a/tests/extmod/ubinascii_a2b_base64.py b/tests/extmod/ubinascii_a2b_base64.py index 97c4519508..58eb0b50b6 100644 --- a/tests/extmod/ubinascii_a2b_base64.py +++ b/tests/extmod/ubinascii_a2b_base64.py @@ -1,7 +1,12 @@ try: - import ubinascii as binascii + try: + import ubinascii as binascii + except ImportError: + import binascii except ImportError: - import binascii + import sys + print("SKIP") + sys.exit() print(binascii.a2b_base64(b'')) print(binascii.a2b_base64(b'Zg==')) diff --git a/tests/extmod/ubinascii_b2a_base64.py b/tests/extmod/ubinascii_b2a_base64.py index fdcaf32dd8..1c0c30311d 100644 --- a/tests/extmod/ubinascii_b2a_base64.py +++ b/tests/extmod/ubinascii_b2a_base64.py @@ -1,7 +1,12 @@ try: - import ubinascii as binascii + try: + import ubinascii as binascii + except ImportError: + import binascii except ImportError: - import binascii + import sys + print("SKIP") + sys.exit() print(binascii.b2a_base64(b'')) print(binascii.b2a_base64(b'f')) diff --git a/tests/extmod/ubinascii_crc32.py b/tests/extmod/ubinascii_crc32.py index 2c40177518..b82c44d6bf 100644 --- a/tests/extmod/ubinascii_crc32.py +++ b/tests/extmod/ubinascii_crc32.py @@ -1,7 +1,13 @@ try: - import ubinascii as binascii + try: + import ubinascii as binascii + except ImportError: + import binascii except ImportError: - import binascii + import sys + print("SKIP") + sys.exit() + try: binascii.crc32 except AttributeError: diff --git a/tests/extmod/ubinascii_hexlify.py b/tests/extmod/ubinascii_hexlify.py index 14c37cb4be..5d70bda96d 100644 --- a/tests/extmod/ubinascii_hexlify.py +++ b/tests/extmod/ubinascii_hexlify.py @@ -1,7 +1,12 @@ try: - import ubinascii as binascii + try: + import ubinascii as binascii + except ImportError: + import binascii except ImportError: - import binascii + import sys + print("SKIP") + sys.exit() print(binascii.hexlify(b'\x00\x01\x02\x03\x04\x05\x06\x07')) print(binascii.hexlify(b'\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f')) diff --git a/tests/extmod/ubinascii_micropython.py b/tests/extmod/ubinascii_micropython.py index d68da3205d..96f566bd14 100644 --- a/tests/extmod/ubinascii_micropython.py +++ b/tests/extmod/ubinascii_micropython.py @@ -1,7 +1,12 @@ try: - import ubinascii as binascii + try: + import ubinascii as binascii + except ImportError: + import binascii except ImportError: - import binascii + import sys + print("SKIP") + sys.exit() # two arguments supported in uPy but not CPython a = binascii.hexlify(b'123', ':') diff --git a/tests/extmod/ubinascii_unhexlify.py b/tests/extmod/ubinascii_unhexlify.py index 99c2c0208d..e669789ba8 100644 --- a/tests/extmod/ubinascii_unhexlify.py +++ b/tests/extmod/ubinascii_unhexlify.py @@ -1,7 +1,12 @@ try: - import ubinascii as binascii + try: + import ubinascii as binascii + except ImportError: + import binascii except ImportError: - import binascii + import sys + print("SKIP") + sys.exit() print(binascii.unhexlify(b'0001020304050607')) print(binascii.unhexlify(b'08090a0b0c0d0e0f')) diff --git a/tests/extmod/uctypes_array_assign_le.py b/tests/extmod/uctypes_array_assign_le.py index 18d63f8bf9..bae467d092 100644 --- a/tests/extmod/uctypes_array_assign_le.py +++ b/tests/extmod/uctypes_array_assign_le.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() desc = { # arr is array at offset 0, of UINT8 elements, array size is 2 diff --git a/tests/extmod/uctypes_array_assign_native_le.py b/tests/extmod/uctypes_array_assign_native_le.py index 632c4d252b..474b7e0f2d 100644 --- a/tests/extmod/uctypes_array_assign_native_le.py +++ b/tests/extmod/uctypes_array_assign_native_le.py @@ -1,5 +1,9 @@ import sys -import uctypes +try: + import uctypes +except ImportError: + print("SKIP") + sys.exit() if sys.byteorder != "little": print("SKIP") diff --git a/tests/extmod/uctypes_bytearray.py b/tests/extmod/uctypes_bytearray.py index 7294b7ea45..bf7845ab26 100644 --- a/tests/extmod/uctypes_bytearray.py +++ b/tests/extmod/uctypes_bytearray.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() desc = { "arr": (uctypes.ARRAY | 0, uctypes.UINT8 | 2), diff --git a/tests/extmod/uctypes_le.py b/tests/extmod/uctypes_le.py index 5ae410b011..829beda58b 100644 --- a/tests/extmod/uctypes_le.py +++ b/tests/extmod/uctypes_le.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() desc = { "s0": uctypes.UINT16 | 0, diff --git a/tests/extmod/uctypes_le_float.py b/tests/extmod/uctypes_le_float.py index c85b75f36c..a61305ba86 100644 --- a/tests/extmod/uctypes_le_float.py +++ b/tests/extmod/uctypes_le_float.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() desc = { "f32": uctypes.FLOAT32 | 0, diff --git a/tests/extmod/uctypes_native_float.py b/tests/extmod/uctypes_native_float.py index 89aac8bf38..80cb54383d 100644 --- a/tests/extmod/uctypes_native_float.py +++ b/tests/extmod/uctypes_native_float.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() desc = { "f32": uctypes.FLOAT32 | 0, diff --git a/tests/extmod/uctypes_native_le.py b/tests/extmod/uctypes_native_le.py index ef0f9f5e95..5900224d41 100644 --- a/tests/extmod/uctypes_native_le.py +++ b/tests/extmod/uctypes_native_le.py @@ -2,7 +2,11 @@ # Codepaths for packed vs native structures are different. This test only works # on little-endian machine (no matter if 32 or 64 bit). import sys -import uctypes +try: + import uctypes +except ImportError: + print("SKIP") + sys.exit() if sys.byteorder != "little": print("SKIP") diff --git a/tests/extmod/uctypes_print.py b/tests/extmod/uctypes_print.py index 71981ce7ea..76a009dc75 100644 --- a/tests/extmod/uctypes_print.py +++ b/tests/extmod/uctypes_print.py @@ -1,6 +1,10 @@ # test printing of uctypes objects - -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() # we use an address of "0" because we just want to print something deterministic # and don't actually need to set/get any values in the struct diff --git a/tests/extmod/uctypes_ptr_le.py b/tests/extmod/uctypes_ptr_le.py index d0216dfb81..e8a6243ce1 100644 --- a/tests/extmod/uctypes_ptr_le.py +++ b/tests/extmod/uctypes_ptr_le.py @@ -1,5 +1,9 @@ import sys -import uctypes +try: + import uctypes +except ImportError: + print("SKIP") + sys.exit() if sys.byteorder != "little": print("SKIP") diff --git a/tests/extmod/uctypes_ptr_native_le.py b/tests/extmod/uctypes_ptr_native_le.py index 6f011c3c2b..9b016c04d5 100644 --- a/tests/extmod/uctypes_ptr_native_le.py +++ b/tests/extmod/uctypes_ptr_native_le.py @@ -1,5 +1,9 @@ import sys -import uctypes +try: + import uctypes +except ImportError: + print("SKIP") + sys.exit() if sys.byteorder != "little": print("SKIP") diff --git a/tests/extmod/uctypes_sizeof.py b/tests/extmod/uctypes_sizeof.py index fcfd8ecd74..266cd06943 100644 --- a/tests/extmod/uctypes_sizeof.py +++ b/tests/extmod/uctypes_sizeof.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() desc = { # arr is array at offset 0, of UINT8 elements, array size is 2 diff --git a/tests/extmod/uctypes_sizeof_native.py b/tests/extmod/uctypes_sizeof_native.py index f830a1f85e..f676c8c6de 100644 --- a/tests/extmod/uctypes_sizeof_native.py +++ b/tests/extmod/uctypes_sizeof_native.py @@ -1,4 +1,9 @@ -import uctypes +try: + import uctypes +except ImportError: + import sys + print("SKIP") + sys.exit() S1 = {} assert uctypes.sizeof(S1) == 0 diff --git a/tests/extmod/uheapq1.py b/tests/extmod/uheapq1.py index e71f817ef8..4b0e5de57e 100644 --- a/tests/extmod/uheapq1.py +++ b/tests/extmod/uheapq1.py @@ -1,7 +1,12 @@ try: import uheapq as heapq except: - import heapq + try: + import heapq + except ImportError: + import sys + print("SKIP") + sys.exit() try: heapq.heappop([]) diff --git a/tests/extmod/ujson_dumps.py b/tests/extmod/ujson_dumps.py index c0ee60d736..4a02f51704 100644 --- a/tests/extmod/ujson_dumps.py +++ b/tests/extmod/ujson_dumps.py @@ -1,7 +1,12 @@ try: import ujson as json except ImportError: - import json + try: + import json + except ImportError: + import sys + print("SKIP") + sys.exit() print(json.dumps(False)) print(json.dumps(True)) diff --git a/tests/extmod/ujson_dumps_extra.py b/tests/extmod/ujson_dumps_extra.py index 0e593c3e93..a52e8224c4 100644 --- a/tests/extmod/ujson_dumps_extra.py +++ b/tests/extmod/ujson_dumps_extra.py @@ -1,5 +1,10 @@ # test uPy ujson behaviour that's not valid in CPy -import ujson +try: + import ujson +except ImportError: + import sys + print("SKIP") + sys.exit() print(ujson.dumps(b'1234')) diff --git a/tests/extmod/ujson_dumps_float.py b/tests/extmod/ujson_dumps_float.py index f6ba5f1139..d949ea6ddd 100644 --- a/tests/extmod/ujson_dumps_float.py +++ b/tests/extmod/ujson_dumps_float.py @@ -1,6 +1,11 @@ try: import ujson as json except ImportError: - import json + try: + import json + except ImportError: + import sys + print("SKIP") + sys.exit() print(json.dumps(1.2)) diff --git a/tests/extmod/ujson_load.py b/tests/extmod/ujson_load.py index bf484a2074..901132a5f8 100644 --- a/tests/extmod/ujson_load.py +++ b/tests/extmod/ujson_load.py @@ -2,8 +2,13 @@ try: from uio import StringIO import ujson as json except: - from io import StringIO - import json + try: + from io import StringIO + import json + except ImportError: + import sys + print("SKIP") + sys.exit() print(json.load(StringIO('null'))) print(json.load(StringIO('"abc\\u0064e"'))) diff --git a/tests/extmod/ujson_loads.py b/tests/extmod/ujson_loads.py index 37576e6ae8..b2e18e3af7 100644 --- a/tests/extmod/ujson_loads.py +++ b/tests/extmod/ujson_loads.py @@ -1,7 +1,12 @@ try: import ujson as json -except: - import json +except ImportError: + try: + import json + except ImportError: + import sys + print("SKIP") + sys.exit() def my_print(o): if isinstance(o, dict): diff --git a/tests/extmod/ujson_loads_float.py b/tests/extmod/ujson_loads_float.py index f5e754608c..b20a412ffb 100644 --- a/tests/extmod/ujson_loads_float.py +++ b/tests/extmod/ujson_loads_float.py @@ -1,7 +1,12 @@ try: import ujson as json -except: - import json +except ImportError: + try: + import json + except ImportError: + import sys + print("SKIP") + sys.exit() def my_print(o): print('%.3f' % o) diff --git a/tests/extmod/urandom_basic.py b/tests/extmod/urandom_basic.py index bf00035bd5..885b8517f4 100644 --- a/tests/extmod/urandom_basic.py +++ b/tests/extmod/urandom_basic.py @@ -1,7 +1,12 @@ try: import urandom as random except ImportError: - import random + try: + import random + except ImportError: + import sys + print("SKIP") + sys.exit() # check getrandbits returns a value within the bit range for b in (1, 2, 3, 4, 16, 32): diff --git a/tests/extmod/urandom_extra.py b/tests/extmod/urandom_extra.py index 004fb10cc4..925dd0dbcb 100644 --- a/tests/extmod/urandom_extra.py +++ b/tests/extmod/urandom_extra.py @@ -1,7 +1,12 @@ try: import urandom as random except ImportError: - import random + try: + import random + except ImportError: + import sys + print("SKIP") + sys.exit() try: random.randint diff --git a/tests/extmod/ure1.py b/tests/extmod/ure1.py index 48537c2ea5..b5edeeace3 100644 --- a/tests/extmod/ure1.py +++ b/tests/extmod/ure1.py @@ -1,7 +1,12 @@ try: import ure as re except ImportError: - import re + try: + import re + except ImportError: + import sys + print("SKIP") + sys.exit() r = re.compile(".+") m = r.match("abc") diff --git a/tests/extmod/ure_debug.py b/tests/extmod/ure_debug.py index 303e1789c0..252df21e30 100644 --- a/tests/extmod/ure_debug.py +++ b/tests/extmod/ure_debug.py @@ -1,3 +1,9 @@ # test printing debugging info when compiling -import ure +try: + import ure +except ImportError: + import sys + print("SKIP") + sys.exit() + ure.compile('^a|b[0-9]\w$', ure.DEBUG) diff --git a/tests/extmod/ure_error.py b/tests/extmod/ure_error.py index 1e9f66a9e5..3f16f9158c 100644 --- a/tests/extmod/ure_error.py +++ b/tests/extmod/ure_error.py @@ -2,8 +2,13 @@ try: import ure as re -except: - import re +except ImportError: + try: + import re + except ImportError: + import sys + print("SKIP") + sys.exit() def test_re(r): try: diff --git a/tests/extmod/ure_group.py b/tests/extmod/ure_group.py index 8078a0a855..98aae2a736 100644 --- a/tests/extmod/ure_group.py +++ b/tests/extmod/ure_group.py @@ -2,8 +2,13 @@ try: import ure as re -except: - import re +except ImportError: + try: + import re + except ImportError: + import sys + print("SKIP") + sys.exit() def print_groups(match): print('----') diff --git a/tests/extmod/ure_namedclass.py b/tests/extmod/ure_namedclass.py index 25f425ce71..e233f17c88 100644 --- a/tests/extmod/ure_namedclass.py +++ b/tests/extmod/ure_namedclass.py @@ -2,8 +2,13 @@ try: import ure as re -except: - import re +except ImportError: + try: + import re + except ImportError: + import sys + print("SKIP") + sys.exit() def print_groups(match): print('----') diff --git a/tests/extmod/ure_split.py b/tests/extmod/ure_split.py index 620fd9052b..1e411c27c5 100644 --- a/tests/extmod/ure_split.py +++ b/tests/extmod/ure_split.py @@ -1,7 +1,12 @@ try: import ure as re except ImportError: - import re + try: + import re + except ImportError: + import sys + print("SKIP") + sys.exit() r = re.compile(" ") s = r.split("a b c foobar") diff --git a/tests/extmod/ure_split_empty.py b/tests/extmod/ure_split_empty.py index 6f31e6dc6c..ad6334ebae 100644 --- a/tests/extmod/ure_split_empty.py +++ b/tests/extmod/ure_split_empty.py @@ -4,7 +4,12 @@ # behaviour will change in a future version. MicroPython just stops # splitting as soon as an empty match is found. -import ure as re +try: + import ure as re +except ImportError: + import sys + print("SKIP") + sys.exit() r = re.compile(" *") s = r.split("a b c foobar") diff --git a/tests/extmod/ure_split_notimpl.py b/tests/extmod/ure_split_notimpl.py index 724e9d43be..eca3ea512e 100644 --- a/tests/extmod/ure_split_notimpl.py +++ b/tests/extmod/ure_split_notimpl.py @@ -1,4 +1,9 @@ -import ure as re +try: + import ure as re +except ImportError: + import sys + print("SKIP") + sys.exit() r = re.compile('( )') try: diff --git a/tests/extmod/uzlib_decompio.py b/tests/extmod/uzlib_decompio.py index 75a6df0ca4..6f07c048c7 100644 --- a/tests/extmod/uzlib_decompio.py +++ b/tests/extmod/uzlib_decompio.py @@ -1,8 +1,10 @@ try: - import zlib -except ImportError: import uzlib as zlib -import uio as io + import uio as io +except ImportError: + import sys + print("SKIP") + sys.exit() # Raw DEFLATE bitstream diff --git a/tests/extmod/uzlib_decompio_gz.py b/tests/extmod/uzlib_decompio_gz.py index c7aac04e8e..5ab25354cd 100644 --- a/tests/extmod/uzlib_decompio_gz.py +++ b/tests/extmod/uzlib_decompio_gz.py @@ -1,8 +1,10 @@ try: - import zlib -except ImportError: import uzlib as zlib -import uio as io + import uio as io +except ImportError: + import sys + print("SKIP") + sys.exit() # gzip bitstream diff --git a/tests/extmod/uzlib_decompress.py b/tests/extmod/uzlib_decompress.py index 6892808cb4..10121ee7e1 100644 --- a/tests/extmod/uzlib_decompress.py +++ b/tests/extmod/uzlib_decompress.py @@ -1,7 +1,12 @@ try: import zlib except ImportError: - import uzlib as zlib + try: + import uzlib as zlib + except ImportError: + import sys + print("SKIP") + sys.exit() PATTERNS = [ # Packed results produced by CPy's zlib.compress() diff --git a/tests/extmod/vfs_fat_oldproto.py b/tests/extmod/vfs_fat_oldproto.py index 03ab76e35f..77d3492125 100644 --- a/tests/extmod/vfs_fat_oldproto.py +++ b/tests/extmod/vfs_fat_oldproto.py @@ -1,9 +1,14 @@ import sys -import uerrno try: - import uos_vfs as uos + import uerrno + try: + import uos_vfs as uos + except ImportError: + import uos except ImportError: - import uos + print("SKIP") + sys.exit() + try: uos.VfsFat except AttributeError: diff --git a/tests/extmod/vfs_fat_ramdisk.py b/tests/extmod/vfs_fat_ramdisk.py index dc4a1c3055..81f9418b28 100644 --- a/tests/extmod/vfs_fat_ramdisk.py +++ b/tests/extmod/vfs_fat_ramdisk.py @@ -1,9 +1,14 @@ import sys -import uerrno try: - import uos_vfs as uos + import uerrno + try: + import uos_vfs as uos + except ImportError: + import uos except ImportError: - import uos + print("SKIP") + sys.exit() + try: uos.VfsFat except AttributeError: From 2847d7431d9334220445e93dc7af707821e80d5e Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 4 Feb 2017 23:33:20 +1100 Subject: [PATCH 33/92] tests/thread: Replace busy waiting loops with a loop that sleeps. Depending on the thread scheduler, a busy-wait loop can hog the CPU and make the tests very slow. So convert such loops to loops that have an explicit sleep, allowing the worker threads to do their job. --- tests/thread/stress_heap.py | 8 ++++++-- tests/thread/thread_lock4.py | 5 +++++ tests/thread/thread_qstr1.py | 8 ++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/thread/stress_heap.py b/tests/thread/stress_heap.py index ac3ebe0491..5482a9ac6f 100644 --- a/tests/thread/stress_heap.py +++ b/tests/thread/stress_heap.py @@ -3,6 +3,10 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +try: + import utime as time +except ImportError: + import time import _thread def last(l): @@ -37,6 +41,6 @@ n_finished = 0 for i in range(n_thread): _thread.start_new_thread(thread_entry, (10000,)) -# busy wait for threads to finish +# wait for threads to finish while n_finished < n_thread: - pass + time.sleep(1) diff --git a/tests/thread/thread_lock4.py b/tests/thread/thread_lock4.py index d77aa24ee8..2f9d42d6b5 100644 --- a/tests/thread/thread_lock4.py +++ b/tests/thread/thread_lock4.py @@ -2,6 +2,10 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +try: + import utime as time +except ImportError: + import time import _thread def fac(n): @@ -39,6 +43,7 @@ while True: with jobs_lock: if len(output) == n_jobs: break + time.sleep(1) # sort and print the results output.sort(key=lambda x: x[0]) diff --git a/tests/thread/thread_qstr1.py b/tests/thread/thread_qstr1.py index c0256316e5..f4136d9646 100644 --- a/tests/thread/thread_qstr1.py +++ b/tests/thread/thread_qstr1.py @@ -2,6 +2,10 @@ # # MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd +try: + import utime as time +except ImportError: + import time import _thread # function to check the interned string @@ -28,8 +32,8 @@ n_qstr_per_thread = 100 # make 1000 for a more stressful test (uses more heap) for i in range(n_thread): _thread.start_new_thread(th, (i * n_qstr_per_thread, n_qstr_per_thread)) -# busy wait for threads to finish +# wait for threads to finish while n_finished < n_thread: - pass + time.sleep(1) print('pass') From 06a12ada48c11a101eb138220cd0e0ca33a612be Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 4 Feb 2017 23:35:08 +1100 Subject: [PATCH 34/92] tests/thread: Add stress-test for creating many threads. --- tests/thread/stress_create.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/thread/stress_create.py diff --git a/tests/thread/stress_create.py b/tests/thread/stress_create.py new file mode 100644 index 0000000000..2399746cca --- /dev/null +++ b/tests/thread/stress_create.py @@ -0,0 +1,22 @@ +# stress test for creating many threads + +try: + import utime as time +except ImportError: + import time +import _thread + +def thread_entry(n): + pass + +thread_num = 0 +while thread_num < 500: + try: + _thread.start_new_thread(thread_entry, (thread_num,)) + thread_num += 1 + except MemoryError: + pass + +# wait for the last threads to terminate +time.sleep(1) +print('done') From 234f07f16cc845128658cc0b39ac681cae5a8fba Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 6 Feb 2017 10:47:20 +1100 Subject: [PATCH 35/92] py/modthread: Use system-provided mutexs for _thread locks. It's more efficient using the system mutexs instead of synthetic ones with a busy-wait loop. The system can do proper scheduling and blocking of the threads waiting on the mutex. --- py/modthread.c | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/py/modthread.c b/py/modthread.c index 6f55281adc..975e7d1f63 100644 --- a/py/modthread.c +++ b/py/modthread.c @@ -44,24 +44,19 @@ /****************************************************************/ // Lock object -// Note: with the GIL enabled we can easily synthesise a lock object STATIC const mp_obj_type_t mp_type_thread_lock; typedef struct _mp_obj_thread_lock_t { mp_obj_base_t base; - #if !MICROPY_PY_THREAD_GIL mp_thread_mutex_t mutex; - #endif volatile bool locked; } mp_obj_thread_lock_t; STATIC mp_obj_thread_lock_t *mp_obj_new_thread_lock(void) { mp_obj_thread_lock_t *self = m_new_obj(mp_obj_thread_lock_t); self->base.type = &mp_type_thread_lock; - #if !MICROPY_PY_THREAD_GIL mp_thread_mutex_init(&self->mutex); - #endif self->locked = false; return self; } @@ -73,20 +68,9 @@ STATIC mp_obj_t thread_lock_acquire(size_t n_args, const mp_obj_t *args) { wait = mp_obj_get_int(args[1]); // TODO support timeout arg } - #if MICROPY_PY_THREAD_GIL - if (self->locked) { - if (!wait) { - return mp_const_false; - } - do { - MP_THREAD_GIL_EXIT(); - MP_THREAD_GIL_ENTER(); - } while (self->locked); - } - self->locked = true; - return mp_const_true; - #else + MP_THREAD_GIL_EXIT(); int ret = mp_thread_mutex_lock(&self->mutex, wait); + MP_THREAD_GIL_ENTER(); if (ret == 0) { return mp_const_false; } else if (ret == 1) { @@ -95,7 +79,6 @@ STATIC mp_obj_t thread_lock_acquire(size_t n_args, const mp_obj_t *args) { } else { mp_raise_OSError(-ret); } - #endif } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(thread_lock_acquire_obj, 1, 3, thread_lock_acquire); @@ -103,9 +86,9 @@ STATIC mp_obj_t thread_lock_release(mp_obj_t self_in) { mp_obj_thread_lock_t *self = MP_OBJ_TO_PTR(self_in); // TODO check if already unlocked self->locked = false; - #if !MICROPY_PY_THREAD_GIL + MP_THREAD_GIL_EXIT(); mp_thread_mutex_unlock(&self->mutex); - #endif + MP_THREAD_GIL_ENTER(); return mp_const_none; } STATIC MP_DEFINE_CONST_FUN_OBJ_1(thread_lock_release_obj, thread_lock_release); From f6c22a06797735e4a65a91491d8373ba951a798b Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 6 Feb 2017 10:50:43 +1100 Subject: [PATCH 36/92] py/vm: Add MICROPY_PY_THREAD_GIL_VM_DIVISOR option. This improves efficiency of GIL release within the VM, by only doing the release after a fixed number of jump-opcodes have executed in the current thread. --- py/mpconfig.h | 6 ++++++ py/vm.c | 20 +++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/py/mpconfig.h b/py/mpconfig.h index 093625a461..1b47d822f1 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -949,6 +949,12 @@ typedef double mp_float_t; #define MICROPY_PY_THREAD_GIL (MICROPY_PY_THREAD) #endif +// Number of VM jump-loops to do before releasing the GIL. +// Set this to 0 to disable the divisor. +#ifndef MICROPY_PY_THREAD_GIL_VM_DIVISOR +#define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32) +#endif + // Extended modules #ifndef MICROPY_PY_UCTYPES diff --git a/py/vm.c b/py/vm.c index f4cfa2cd6f..fb59bf9cff 100644 --- a/py/vm.c +++ b/py/vm.c @@ -169,6 +169,12 @@ run_code_state: ; volatile bool currently_in_except_block = MP_TAGPTR_TAG0(code_state->exc_sp); // 0 or 1, to detect nested exceptions mp_exc_stack_t *volatile exc_sp = MP_TAGPTR_PTR(code_state->exc_sp); // stack grows up, exc_sp points to top of stack + #if MICROPY_PY_THREAD_GIL && MICROPY_PY_THREAD_GIL_VM_DIVISOR + // This needs to be volatile and outside the VM loop so it persists across handling + // of any exceptions. Otherwise it's possible that the VM never gives up the GIL. + volatile int gil_divisor = MICROPY_PY_THREAD_GIL_VM_DIVISOR; + #endif + // outer exception handling loop for (;;) { nlr_buf_t nlr; @@ -1243,9 +1249,17 @@ pending_exception_check: RAISE(obj); } - // TODO make GIL release more efficient - MP_THREAD_GIL_EXIT(); - MP_THREAD_GIL_ENTER(); + #if MICROPY_PY_THREAD_GIL + #if MICROPY_PY_THREAD_GIL_VM_DIVISOR + if (--gil_divisor == 0) { + gil_divisor = MICROPY_PY_THREAD_GIL_VM_DIVISOR; + #else + { + #endif + MP_THREAD_GIL_EXIT(); + MP_THREAD_GIL_ENTER(); + } + #endif } // for loop From 05a4859585c4e0a55fca2e7467ba70da6453fdcb Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 6 Feb 2017 15:13:30 +1100 Subject: [PATCH 37/92] stmhal: Implement a proper thread scheduler. This patch changes the threading implementation from simple round-robin with busy waits on mutexs, to proper scheduling whereby threads that are waiting on a mutex are only scheduled when the mutex becomes available. --- stmhal/main.c | 4 + stmhal/modmachine.c | 5 ++ stmhal/mpconfigport.h | 14 ++++ stmhal/mpthreadport.c | 35 +-------- stmhal/mpthreadport.h | 14 +++- stmhal/pybthread.c | 179 ++++++++++++++++++++++++++++++++++++------ stmhal/pybthread.h | 26 ++++-- stmhal/stm32_it.c | 12 ++- stmhal/systick.c | 10 +++ 9 files changed, 236 insertions(+), 63 deletions(-) diff --git a/stmhal/main.c b/stmhal/main.c index 7bfdc52c3e..bcc429df2f 100644 --- a/stmhal/main.c +++ b/stmhal/main.c @@ -695,6 +695,10 @@ soft_reset_exit: can_deinit(); #endif + #if MICROPY_PY_THREAD + pyb_thread_deinit(); + #endif + first_soft_reset = false; goto soft_reset; } diff --git a/stmhal/modmachine.c b/stmhal/modmachine.c index 4a3fe1ec9a..16c50079d1 100644 --- a/stmhal/modmachine.c +++ b/stmhal/modmachine.c @@ -41,6 +41,7 @@ #include "extmod/vfs_fat.h" #include "gccollect.h" #include "irq.h" +#include "pybthread.h" #include "rng.h" #include "storage.h" #include "pin.h" @@ -159,6 +160,10 @@ STATIC mp_obj_t machine_info(mp_uint_t n_args, const mp_obj_t *args) { } } + #if MICROPY_PY_THREAD + pyb_thread_dump(); + #endif + if (n_args == 1) { // arg given means dump gc allocation table gc_dump_alloc_table(); diff --git a/stmhal/mpconfigport.h b/stmhal/mpconfigport.h index 5828f07c7d..38a5ac5da9 100644 --- a/stmhal/mpconfigport.h +++ b/stmhal/mpconfigport.h @@ -299,7 +299,21 @@ static inline mp_uint_t disable_irq(void) { #define MICROPY_BEGIN_ATOMIC_SECTION() disable_irq() #define MICROPY_END_ATOMIC_SECTION(state) enable_irq(state) + +#if MICROPY_PY_THREAD +#define MICROPY_EVENT_POLL_HOOK \ + do { \ + if (pyb_thread_enabled) { \ + MP_THREAD_GIL_EXIT(); \ + pyb_thread_yield(); \ + MP_THREAD_GIL_ENTER(); \ + } else { \ + __WFI(); \ + } \ + } while (0); +#else #define MICROPY_EVENT_POLL_HOOK __WFI(); +#endif // There is no classical C heap in bare-metal ports, only Python // garbage-collected heap. For completeness, emulate C heap via diff --git a/stmhal/mpthreadport.c b/stmhal/mpthreadport.c index 97c19647cb..d7c5b569bb 100644 --- a/stmhal/mpthreadport.c +++ b/stmhal/mpthreadport.c @@ -44,15 +44,13 @@ void mp_thread_init(void) { void mp_thread_gc_others(void) { mp_thread_mutex_lock(&thread_mutex, 1); - gc_collect_root((void**)&pyb_thread_cur, 1); - for (pyb_thread_t *th = pyb_thread_cur;; th = th->next) { + for (pyb_thread_t *th = pyb_thread_all; th != NULL; th = th->all_next) { + gc_collect_root((void**)&th, 1); gc_collect_root(&th->arg, 1); + gc_collect_root(&th->stack, 1); if (th != pyb_thread_cur) { gc_collect_root(th->stack, th->stack_len); } - if (th->next == pyb_thread_cur) { - break; - } } mp_thread_mutex_unlock(&thread_mutex); } @@ -93,31 +91,4 @@ void mp_thread_start(void) { void mp_thread_finish(void) { } -void mp_thread_mutex_init(mp_thread_mutex_t *mutex) { - *mutex = 0; -} - -int mp_thread_mutex_lock(mp_thread_mutex_t *mutex, int wait) { - uint32_t irq_state = disable_irq(); - if (*mutex) { - // mutex is locked - if (!wait) { - enable_irq(irq_state); - return 0; // failed to lock mutex - } - while (*mutex) { - enable_irq(irq_state); - pyb_thread_yield(); - irq_state = disable_irq(); - } - } - *mutex = 1; - enable_irq(irq_state); - return 1; // have mutex -} - -void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex) { - *mutex = 0; -} - #endif // MICROPY_PY_THREAD diff --git a/stmhal/mpthreadport.h b/stmhal/mpthreadport.h index 4fef323eb5..3d8b4ef011 100644 --- a/stmhal/mpthreadport.h +++ b/stmhal/mpthreadport.h @@ -29,7 +29,7 @@ #include "py/mpthread.h" #include "pybthread.h" -typedef uint32_t mp_thread_mutex_t; +typedef pyb_mutex_t mp_thread_mutex_t; void mp_thread_init(void); void mp_thread_gc_others(void); @@ -42,4 +42,16 @@ static inline struct _mp_state_thread_t *mp_thread_get_state(void) { return pyb_thread_get_local(); } +static inline void mp_thread_mutex_init(mp_thread_mutex_t *m) { + pyb_mutex_init(m); +} + +static inline int mp_thread_mutex_lock(mp_thread_mutex_t *m, int wait) { + return pyb_mutex_lock(m, wait); +} + +static inline void mp_thread_mutex_unlock(mp_thread_mutex_t *m) { + pyb_mutex_unlock(m); +} + #endif // __MICROPY_INCLUDED_STMHAL_MPTHREADPORT_H__ diff --git a/stmhal/pybthread.c b/stmhal/pybthread.c index 9f9f82a451..51e9a738d0 100644 --- a/stmhal/pybthread.c +++ b/stmhal/pybthread.c @@ -34,29 +34,80 @@ #if MICROPY_PY_THREAD -int pyb_thread_enabled; -pyb_thread_t *pyb_thread_cur; +#define PYB_MUTEX_UNLOCKED ((void*)0) +#define PYB_MUTEX_LOCKED ((void*)1) + +extern void __fatal_error(const char*); + +volatile int pyb_thread_enabled; +pyb_thread_t *volatile pyb_thread_all; +pyb_thread_t *volatile pyb_thread_cur; + +static inline void pyb_thread_add_to_runable(pyb_thread_t *thread) { + thread->run_prev = pyb_thread_cur->run_prev; + thread->run_next = pyb_thread_cur; + pyb_thread_cur->run_prev->run_next = thread; + pyb_thread_cur->run_prev = thread; +} + +static inline void pyb_thread_remove_from_runable(pyb_thread_t *thread) { + if (thread->run_next == thread) { + __fatal_error("deadlock"); + } + thread->run_prev->run_next = thread->run_next; + thread->run_next->run_prev = thread->run_prev; +} void pyb_thread_init(pyb_thread_t *thread) { + pyb_thread_enabled = 0; + pyb_thread_all = thread; pyb_thread_cur = thread; - pyb_thread_cur->sp = NULL; // will be set when this thread switches out - pyb_thread_cur->local_state = 0; // will be set by mp_thread_init - pyb_thread_cur->arg = NULL; - pyb_thread_cur->stack = &_heap_end; - pyb_thread_cur->stack_len = ((uint32_t)&_estack - (uint32_t)&_heap_end) / sizeof(uint32_t); - pyb_thread_cur->prev = thread; - pyb_thread_cur->next = thread; + thread->sp = NULL; // will be set when this thread switches out + thread->local_state = 0; // will be set by mp_thread_init + thread->arg = NULL; + thread->stack = &_heap_end; + thread->stack_len = ((uint32_t)&_estack - (uint32_t)&_heap_end) / sizeof(uint32_t); + thread->all_next = NULL; + thread->run_prev = thread; + thread->run_next = thread; + thread->queue_next = NULL; +} + +void pyb_thread_deinit() { + uint32_t irq_state = disable_irq(); + pyb_thread_enabled = 0; + pyb_thread_all = pyb_thread_cur; + pyb_thread_cur->all_next = NULL; + pyb_thread_cur->run_prev = pyb_thread_cur; + pyb_thread_cur->run_next = pyb_thread_cur; + enable_irq(irq_state); } STATIC void pyb_thread_terminate(void) { - uint32_t irq_state = raise_irq_pri(IRQ_PRI_PENDSV); - pyb_thread_cur->prev->next = pyb_thread_cur->next; - pyb_thread_cur->next->prev = pyb_thread_cur->prev; - if (pyb_thread_cur->next == pyb_thread_cur->prev) { + uint32_t irq_state = disable_irq(); + pyb_thread_t *thread = pyb_thread_cur; + // take current thread off the run list + pyb_thread_remove_from_runable(thread); + // take current thread off the list of all threads + for (pyb_thread_t **n = (pyb_thread_t**)&pyb_thread_all;; n = &(*n)->all_next) { + if (*n == thread) { + *n = thread->all_next; + break; + } + } + // clean pointers as much as possible to help GC + thread->all_next = NULL; + thread->queue_next = NULL; + thread->stack = NULL; + if (pyb_thread_all->all_next == NULL) { + // only 1 thread left pyb_thread_enabled = 0; } - restore_irq_pri(irq_state); - pyb_thread_yield(); // should not return + // thread switch will occur after we enable irqs + SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; + enable_irq(irq_state); + // should not return + __fatal_error("could not terminate"); } uint32_t pyb_thread_new(pyb_thread_t *thread, void *stack, size_t stack_len, void *entry, void *arg) { @@ -77,21 +128,105 @@ uint32_t pyb_thread_new(pyb_thread_t *thread, void *stack, size_t stack_len, voi thread->arg = arg; thread->stack = stack; thread->stack_len = stack_len; - uint32_t irq_state = raise_irq_pri(IRQ_PRI_PENDSV); + thread->queue_next = NULL; + uint32_t irq_state = disable_irq(); pyb_thread_enabled = 1; - thread->next = pyb_thread_cur->next; - thread->prev = pyb_thread_cur; - pyb_thread_cur->next->prev = thread; - pyb_thread_cur->next = thread; - restore_irq_pri(irq_state); + thread->all_next = pyb_thread_all; + pyb_thread_all = thread; + pyb_thread_add_to_runable(thread); + enable_irq(irq_state); return (uint32_t)thread; // success } +void pyb_thread_dump(void) { + if (!pyb_thread_enabled) { + printf("THREAD: only main thread\n"); + } else { + printf("THREAD:\n"); + for (pyb_thread_t *th = pyb_thread_all; th != NULL; th = th->all_next) { + bool runable = false; + for (pyb_thread_t *th2 = pyb_thread_cur;; th2 = th2->run_next) { + if (th == th2) { + runable = true; + break; + } + if (th2->run_next == pyb_thread_cur) { + break; + } + } + printf(" id=%p sp=%p sz=%u", th, th->stack, th->stack_len); + if (runable) { + printf(" (runable)"); + } + printf("\n"); + } + } +} + // should only be called from pendsv_isr_handler void *pyb_thread_next(void *sp) { pyb_thread_cur->sp = sp; - pyb_thread_cur = pyb_thread_cur->next; + pyb_thread_cur = pyb_thread_cur->run_next; + pyb_thread_cur->timeslice = 4; // in milliseconds return pyb_thread_cur->sp; } +void pyb_mutex_init(pyb_mutex_t *m) { + *m = PYB_MUTEX_UNLOCKED; +} + +int pyb_mutex_lock(pyb_mutex_t *m, int wait) { + uint32_t irq_state = disable_irq(); + if (*m == PYB_MUTEX_UNLOCKED) { + // mutex is available + *m = PYB_MUTEX_LOCKED; + enable_irq(irq_state); + } else { + // mutex is locked + if (!wait) { + enable_irq(irq_state); + return 0; // failed to lock mutex + } + if (*m == PYB_MUTEX_LOCKED) { + *m = pyb_thread_cur; + } else { + for (pyb_thread_t *n = *m;; n = n->queue_next) { + if (n->queue_next == NULL) { + n->queue_next = pyb_thread_cur; + break; + } + } + } + pyb_thread_cur->queue_next = NULL; + // take current thread off the run list + pyb_thread_remove_from_runable(pyb_thread_cur); + // thread switch will occur after we enable irqs + SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; + enable_irq(irq_state); + // when we come back we have the mutex + } + return 1; // have mutex +} + +void pyb_mutex_unlock(pyb_mutex_t *m) { + uint32_t irq_state = disable_irq(); + if (*m == PYB_MUTEX_LOCKED) { + // no threads are blocked on the mutex + *m = PYB_MUTEX_UNLOCKED; + } else { + // at least one thread is blocked on this mutex + pyb_thread_t *th = *m; + if (th->queue_next == NULL) { + // no other threads are blocked + *m = PYB_MUTEX_LOCKED; + } else { + // at least one other thread is still blocked + *m = th->queue_next; + } + // put unblocked thread on runable list + pyb_thread_add_to_runable(th); + } + enable_irq(irq_state); +} + #endif // MICROPY_PY_THREAD diff --git a/stmhal/pybthread.h b/stmhal/pybthread.h index d4310c66a5..6edb2400e2 100644 --- a/stmhal/pybthread.h +++ b/stmhal/pybthread.h @@ -33,15 +33,23 @@ typedef struct _pyb_thread_t { void *arg; // thread Python args, a GC root pointer void *stack; // pointer to the stack size_t stack_len; // number of words in the stack - struct _pyb_thread_t *prev; - struct _pyb_thread_t *next; + uint32_t timeslice; + struct _pyb_thread_t *all_next; + struct _pyb_thread_t *run_prev; + struct _pyb_thread_t *run_next; + struct _pyb_thread_t *queue_next; } pyb_thread_t; -extern int pyb_thread_enabled; -extern pyb_thread_t *pyb_thread_cur; +typedef pyb_thread_t *pyb_mutex_t; + +extern volatile int pyb_thread_enabled; +extern pyb_thread_t *volatile pyb_thread_all; +extern pyb_thread_t *volatile pyb_thread_cur; void pyb_thread_init(pyb_thread_t *th); +void pyb_thread_deinit(); uint32_t pyb_thread_new(pyb_thread_t *th, void *stack, size_t stack_len, void *entry, void *arg); +void pyb_thread_dump(void); static inline uint32_t pyb_thread_get_id(void) { return (uint32_t)pyb_thread_cur; @@ -56,7 +64,15 @@ static inline void *pyb_thread_get_local(void) { } static inline void pyb_thread_yield(void) { - SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; + if (pyb_thread_cur->run_next == pyb_thread_cur) { + __WFI(); + } else { + SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; + } } +void pyb_mutex_init(pyb_mutex_t *m); +int pyb_mutex_lock(pyb_mutex_t *m, int wait); +void pyb_mutex_unlock(pyb_mutex_t *m); + #endif // MICROPY_INCLUDED_STMHAL_PYBTHREAD_H diff --git a/stmhal/stm32_it.c b/stmhal/stm32_it.c index 4152050a99..d8fcc59ce5 100644 --- a/stmhal/stm32_it.c +++ b/stmhal/stm32_it.c @@ -70,6 +70,7 @@ #include "stm32_it.h" #include STM32_HAL_H +#include "py/mpstate.h" #include "py/obj.h" #include "py/mphal.h" #include "pendsv.h" @@ -315,9 +316,14 @@ void SysTick_Handler(void) { } #if MICROPY_PY_THREAD - // signal a thread switch at 4ms=250Hz - if (pyb_thread_enabled && (uwTick & 0x03) == 0x03) { - SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; + if (pyb_thread_enabled) { + if (pyb_thread_cur->timeslice == 0) { + if (pyb_thread_cur->run_next != pyb_thread_cur) { + SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; + } + } else { + --pyb_thread_cur->timeslice; + } } #endif } diff --git a/stmhal/systick.c b/stmhal/systick.c index aed5b96901..ade05d74d7 100644 --- a/stmhal/systick.c +++ b/stmhal/systick.c @@ -29,9 +29,11 @@ #include "py/obj.h" #include "irq.h" #include "systick.h" +#include "pybthread.h" // We provide our own version of HAL_Delay that calls __WFI while waiting, in // order to reduce power consumption. +// Note: Upon entering this function we may or may not have the GIL. void HAL_Delay(uint32_t Delay) { if (query_irq() == IRQ_STATE_ENABLED) { // IRQs enabled, so can use systick counter to do the delay @@ -40,7 +42,15 @@ void HAL_Delay(uint32_t Delay) { // Wraparound of tick is taken care of by 2's complement arithmetic. while (uwTick - start < Delay) { // Enter sleep mode, waiting for (at least) the SysTick interrupt. + #if MICROPY_PY_THREAD + if (pyb_thread_enabled) { + pyb_thread_yield(); + } else { + __WFI(); + } + #else __WFI(); + #endif } } else { // IRQs disabled, so need to use a busy loop for the delay. From e5cc681cb1b5163b9ae3453df85344326baf9759 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 16:39:30 +1100 Subject: [PATCH 38/92] stmhal: Use generic interrupt char code. --- stmhal/Makefile | 1 + stmhal/mphalport.c | 4 ---- stmhal/pendsv.c | 7 ++++--- stmhal/pendsv.h | 2 +- stmhal/usb.c | 14 +++----------- stmhal/usb.h | 1 - stmhal/usbd_cdc_interface.c | 22 +++++----------------- stmhal/usbd_cdc_interface.h | 1 - 8 files changed, 14 insertions(+), 38 deletions(-) diff --git a/stmhal/Makefile b/stmhal/Makefile index edd5e06193..76579d1305 100644 --- a/stmhal/Makefile +++ b/stmhal/Makefile @@ -112,6 +112,7 @@ SRC_LIB = $(addprefix lib/,\ netutils/netutils.c \ timeutils/timeutils.c \ utils/pyexec.c \ + utils/interrupt_char.c \ ) DRIVERS_SRC_C = $(addprefix drivers/,\ diff --git a/stmhal/mphalport.c b/stmhal/mphalport.c index ca8e1c1bde..64c1164cc8 100644 --- a/stmhal/mphalport.c +++ b/stmhal/mphalport.c @@ -21,10 +21,6 @@ NORETURN void mp_hal_raise(HAL_StatusTypeDef status) { mp_raise_OSError(mp_hal_status_to_errno_table[status]); } -void mp_hal_set_interrupt_char(int c) { - usb_vcp_set_interrupt_char(c); -} - int mp_hal_stdin_rx_chr(void) { for (;;) { #if 0 diff --git a/stmhal/pendsv.c b/stmhal/pendsv.c index e6df84b29a..4c2a14de1b 100644 --- a/stmhal/pendsv.c +++ b/stmhal/pendsv.c @@ -29,6 +29,7 @@ #include "py/mpstate.h" #include "py/runtime.h" +#include "lib/utils/interrupt_char.h" #include "pendsv.h" #include "irq.h" @@ -52,12 +53,12 @@ void pendsv_init(void) { // PENDSV feature. This will wait until all interrupts are finished then raise // the given exception object using nlr_jump in the context of the top-level // thread. -void pendsv_nlr_jump(void *o) { +void pendsv_kbd_intr(void) { if (MP_STATE_VM(mp_pending_exception) == MP_OBJ_NULL) { - MP_STATE_VM(mp_pending_exception) = o; + mp_keyboard_interrupt(); } else { MP_STATE_VM(mp_pending_exception) = MP_OBJ_NULL; - pendsv_object = o; + pendsv_object = &MP_STATE_VM(mp_kbd_exception); SCB->ICSR = SCB_ICSR_PENDSVSET_Msk; } } diff --git a/stmhal/pendsv.h b/stmhal/pendsv.h index 7886d9f984..77c78d4c18 100644 --- a/stmhal/pendsv.h +++ b/stmhal/pendsv.h @@ -25,7 +25,7 @@ */ void pendsv_init(void); -void pendsv_nlr_jump(void *val); +void pendsv_kbd_intr(void); // since we play tricks with the stack, the compiler must not generate a // prelude for this function diff --git a/stmhal/usb.c b/stmhal/usb.c index 7eb3f179a9..c413ce4bac 100644 --- a/stmhal/usb.c +++ b/stmhal/usb.c @@ -38,6 +38,7 @@ #include "py/runtime.h" #include "py/stream.h" #include "py/mperrno.h" +#include "py/mphal.h" #include "bufhelper.h" #include "usb.h" @@ -96,7 +97,7 @@ const mp_obj_tuple_t pyb_usb_hid_keyboard_obj = { }; void pyb_usb_init0(void) { - USBD_CDC_SetInterrupt(-1); + mp_hal_set_interrupt_char(-1); MP_STATE_PORT(pyb_hid_report_desc) = MP_OBJ_NULL; } @@ -141,15 +142,6 @@ bool usb_vcp_is_enabled(void) { return (pyb_usb_flags & PYB_USB_FLAG_DEV_ENABLED) != 0; } -void usb_vcp_set_interrupt_char(int c) { - if (pyb_usb_flags & PYB_USB_FLAG_DEV_ENABLED) { - if (c != -1) { - mp_obj_exception_clear_traceback(MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_kbd_exception))); - } - USBD_CDC_SetInterrupt(c); - } -} - int usb_vcp_recv_byte(uint8_t *c) { return USBD_CDC_Rx(c, 1, 0); } @@ -364,7 +356,7 @@ STATIC mp_obj_t pyb_usb_vcp_make_new(const mp_obj_type_t *type, size_t n_args, s } STATIC mp_obj_t pyb_usb_vcp_setinterrupt(mp_obj_t self_in, mp_obj_t int_chr_in) { - usb_vcp_set_interrupt_char(mp_obj_get_int(int_chr_in)); + mp_hal_set_interrupt_char(mp_obj_get_int(int_chr_in)); return mp_const_none; } STATIC MP_DEFINE_CONST_FUN_OBJ_2(pyb_usb_vcp_setinterrupt_obj, pyb_usb_vcp_setinterrupt); diff --git a/stmhal/usb.h b/stmhal/usb.h index e153f0c6b6..bc2b91c3db 100644 --- a/stmhal/usb.h +++ b/stmhal/usb.h @@ -61,7 +61,6 @@ void pyb_usb_init0(void); bool pyb_usb_dev_init(uint16_t vid, uint16_t pid, usb_device_mode_t mode, USBD_HID_ModeInfoTypeDef *hid_info); void pyb_usb_dev_deinit(void); bool usb_vcp_is_enabled(void); -void usb_vcp_set_interrupt_char(int c); int usb_vcp_recv_byte(uint8_t *c); // if a byte is available, return 1 and put the byte in *c, else return 0 void usb_vcp_send_strn(const char* str, int len); void usb_vcp_send_strn_cooked(const char *str, int len); diff --git a/stmhal/usbd_cdc_interface.c b/stmhal/usbd_cdc_interface.c index 1f46b9dcc1..1c12cdc1c8 100644 --- a/stmhal/usbd_cdc_interface.c +++ b/stmhal/usbd_cdc_interface.c @@ -43,6 +43,7 @@ #include "py/mpstate.h" #include "py/obj.h" +#include "lib/utils/interrupt_char.h" #include "irq.h" #include "timer.h" #include "usb.h" @@ -79,8 +80,6 @@ static uint16_t UserTxBufPtrOutShadow = 0; // shadow of above static uint8_t UserTxBufPtrWaitCount = 0; // used to implement a timeout waiting for low-level USB driver static uint8_t UserTxNeedEmptyPacket = 0; // used to flush the USB IN endpoint if the last packet was exactly the endpoint packet size -static int user_interrupt_char = -1; - /* Private function prototypes -----------------------------------------------*/ static int8_t CDC_Itf_Init (void); static int8_t CDC_Itf_DeInit (void); @@ -147,13 +146,6 @@ static int8_t CDC_Itf_Init(void) UserRxBufCur = 0; UserRxBufLen = 0; - /* NOTE: we cannot reset these here, because USBD_CDC_SetInterrupt - * may be called before this init function to set these values. - * This can happen if the USB enumeration occurs after the call to - * USBD_CDC_SetInterrupt. - user_interrupt_char = -1; - */ - return (USBD_OK); } @@ -339,7 +331,7 @@ static int8_t CDC_Itf_Receive(uint8_t* Buf, uint32_t *Len) { uint32_t delta_len; - if (user_interrupt_char == -1) { + if (mp_interrupt_char == -1) { // no special interrupt character delta_len = *Len; @@ -350,10 +342,10 @@ static int8_t CDC_Itf_Receive(uint8_t* Buf, uint32_t *Len) { uint8_t *src = Buf; uint8_t *buf_top = Buf + *Len; for (; src < buf_top; src++) { - if (*src == user_interrupt_char) { + if (*src == mp_interrupt_char) { char_found = true; - // raise exception when interrupts are finished - pendsv_nlr_jump(&MP_STATE_VM(mp_kbd_exception)); + // raise KeyboardInterrupt when interrupts are finished + pendsv_kbd_intr(); } else { if (char_found) { *dest = *src; @@ -385,10 +377,6 @@ int USBD_CDC_IsConnected(void) { return dev_is_connected; } -void USBD_CDC_SetInterrupt(int chr) { - user_interrupt_char = chr; -} - int USBD_CDC_TxHalfEmpty(void) { int32_t tx_waiting = (int32_t)UserTxBufPtrIn - (int32_t)UserTxBufPtrOut; if (tx_waiting < 0) { diff --git a/stmhal/usbd_cdc_interface.h b/stmhal/usbd_cdc_interface.h index 2ea1a42c4a..d96861a7e5 100644 --- a/stmhal/usbd_cdc_interface.h +++ b/stmhal/usbd_cdc_interface.h @@ -32,7 +32,6 @@ extern const USBD_CDC_ItfTypeDef USBD_CDC_fops; int USBD_CDC_IsConnected(void); -void USBD_CDC_SetInterrupt(int chr); int USBD_CDC_TxHalfEmpty(void); int USBD_CDC_Tx(const uint8_t *buf, uint32_t len, uint32_t timeout); From eb101a2701e5e2159e82a24aedad8911104ef9be Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 13:12:23 +0300 Subject: [PATCH 39/92] examples/embedding/README: Convert to markdown, grammar and clarity fixes. --- examples/embedding/{README => README.md} | 33 ++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) rename examples/embedding/{README => README.md} (58%) diff --git a/examples/embedding/README b/examples/embedding/README.md similarity index 58% rename from examples/embedding/README rename to examples/embedding/README.md index 0475e8739a..989ce1fc8f 100644 --- a/examples/embedding/README +++ b/examples/embedding/README.md @@ -11,7 +11,7 @@ Python statement which prints to the standard output. Building the example -------------------- -Build the example is as simple as running: +Building the example is as simple as running: make @@ -20,37 +20,38 @@ It's worth to trace what's happening behind the scenes though: 1. As a first step, a MicroPython library is built. This is handled by a seperate makefile, Makefile.upylib. It is more or less complex, but the good news is that you won't need to change anything in it, just use it -as is, the main Makefile shows how. What may need editing though is -MicroPython configuration file. MicroPython is highly configurable, so +as is, the main Makefile shows how. What may require editing though is +a MicroPython configuration file. MicroPython is highly configurable, so you would need to build a library suiting your application well, while not bloating its size. Check the options in the file "mpconfigport.h". -Included is a copy of "minimal" Unix port, which should be good start -for minimal embedding. For list of all available options, see py/mpconfig.h. +Included is a copy of the "minimal" Unix port, which should be a good start +for minimal embedding. For the list of all available options, see +py/mpconfig.h. -2. Once the library is built, your application is compiled and linked with -the MicroPython library produced in the previous step. The main Makefile -is very simple and shows that changes you would need to do to your -application's Makefile (or other build configuration) are also simple: +2. Once the MicroPython library is built, your application is compiled +and linked it. The main Makefile is very simple and shows that the changes +you would need to do to your application's Makefile (or other build +configuration) are also simple: -a) You would need to use C99 standard (you're using 15+ years old standard -already, not a 25+ years old one, right?). +a) You would need to use C99 standard (you're using this 15+ years old +standard already, not a 25+ years old one, right?). -b) You need to provide path to MicroPython's top-level dir, for includes. +b) You need to provide a path to MicroPython's top-level dir, for includes. c) You need to include -DNO_QSTR compile-time flag. -d) Otherwise, just link with micropython library produced in step 1. +d) Otherwise, just link with the MicroPython library produced in step 1. Out of tree build ----------------- -This example set up to work out of the box, being part of the MicroPython +This example is set up to work out of the box, being part of the MicroPython tree. Your application of course will be outside of its tree, but the only thing you need to do is to pass MPTOP variable pointing to MicroPython directory to both Makefiles (in this example, the main Makefile -automatically pass it to Makefile.upylib; in your own Makefile, don't forget -to use suitable value). +automatically passes it to Makefile.upylib; in your own Makefile, don't forget +to use a suitable value). A practical way to embed MicroPython in your application is to include it as a git submodule. Suppose you included it as libs/micropython. Then in From b737c9cbc823b012808ae08c8b285b4e1543703a Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 17:05:27 +0300 Subject: [PATCH 40/92] tests/gen_yield_from_close: Use range() instead of reversed(). As a "more basic" builtin iterator, present even in smaller ports. --- tests/basics/gen_yield_from_close.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basics/gen_yield_from_close.py b/tests/basics/gen_yield_from_close.py index d66691ff9c..8339861056 100644 --- a/tests/basics/gen_yield_from_close.py +++ b/tests/basics/gen_yield_from_close.py @@ -102,7 +102,7 @@ except RuntimeError: # case where close is propagated up to a built-in iterator def gen8(): - g = reversed([2, 1]) + g = range(2) yield from g g = gen8() print(next(g)) From f980c70997eb9748ed169d486489ab3d0d2002af Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 18:11:16 +0300 Subject: [PATCH 41/92] tests/basic/: Make various tests skippable. To run the testsuite on small ports. --- tests/basics/iter_of_iter.py | 3 +- tests/basics/map.py | 2 +- tests/basics/memoryview1.py | 6 ++ tests/basics/memoryview2.py | 9 +- tests/basics/memoryview_gc.py | 6 ++ tests/basics/namedtuple1.py | 9 +- tests/basics/object_new.py | 8 ++ tests/basics/op_error.py | 1 - tests/basics/op_error_memoryview.py | 19 ++++ tests/basics/set_iter_of_iter.py | 2 + tests/basics/special_methods.py | 40 +------- tests/basics/special_methods2.py | 145 +++++++++++++++++++++++++++ tests/basics/subclass_classmethod.py | 7 ++ tests/basics/sys1.py | 14 ++- 14 files changed, 222 insertions(+), 49 deletions(-) create mode 100644 tests/basics/op_error_memoryview.py create mode 100644 tests/basics/set_iter_of_iter.py create mode 100644 tests/basics/special_methods2.py diff --git a/tests/basics/iter_of_iter.py b/tests/basics/iter_of_iter.py index 70282aa97e..d775b6a44e 100644 --- a/tests/basics/iter_of_iter.py +++ b/tests/basics/iter_of_iter.py @@ -4,5 +4,4 @@ i = iter(iter([1, 2, 3])) print(list(i)) i = iter(iter({1:2, 3:4, 5:6})) print(sorted(i)) -i = iter(iter({1, 2, 3})) -print(sorted(i)) +# set, see set_iter_of_iter.py diff --git a/tests/basics/map.py b/tests/basics/map.py index 62dca44ede..8fce352c28 100644 --- a/tests/basics/map.py +++ b/tests/basics/map.py @@ -1,4 +1,4 @@ print(list(map(lambda x: x & 1, range(-3, 4)))) print(list(map(abs, range(-3, 4)))) -print(list(map(set, [[i] for i in range(-3, 4)]))) +print(list(map(tuple, [[i] for i in range(-3, 4)]))) print(list(map(pow, range(4), range(4)))) diff --git a/tests/basics/memoryview1.py b/tests/basics/memoryview1.py index 1cd411195d..019a1179f8 100644 --- a/tests/basics/memoryview1.py +++ b/tests/basics/memoryview1.py @@ -1,4 +1,10 @@ # test memoryview +try: + memoryview +except: + import sys + print("SKIP") + sys.exit() # test reading from bytes b = b'1234' diff --git a/tests/basics/memoryview2.py b/tests/basics/memoryview2.py index 5117d7a680..edb7c9e640 100644 --- a/tests/basics/memoryview2.py +++ b/tests/basics/memoryview2.py @@ -1,6 +1,11 @@ # test memoryview accessing maximum values for signed/unsigned elements - -from array import array +try: + from array import array + memoryview +except: + import sys + print("SKIP") + sys.exit() print(list(memoryview(b'\x7f\x80\x81\xff'))) print(list(memoryview(array('b', [0x7f, -0x80])))) diff --git a/tests/basics/memoryview_gc.py b/tests/basics/memoryview_gc.py index a1e4baad47..9d4857e362 100644 --- a/tests/basics/memoryview_gc.py +++ b/tests/basics/memoryview_gc.py @@ -1,4 +1,10 @@ # test memoryview retains pointer to original object/buffer +try: + memoryview +except: + import sys + print("SKIP") + sys.exit() b = bytearray(10) m = memoryview(b)[1:] diff --git a/tests/basics/namedtuple1.py b/tests/basics/namedtuple1.py index 346e32fbfc..132dcf96b3 100644 --- a/tests/basics/namedtuple1.py +++ b/tests/basics/namedtuple1.py @@ -1,7 +1,12 @@ try: - from collections import namedtuple + try: + from collections import namedtuple + except ImportError: + from ucollections import namedtuple except ImportError: - from ucollections import namedtuple + import sys + print("SKIP") + sys.exit() T = namedtuple("Tup", ["foo", "bar"]) # CPython prints fully qualified name, what we don't bother to do so far diff --git a/tests/basics/object_new.py b/tests/basics/object_new.py index befb5bfc27..568feccda4 100644 --- a/tests/basics/object_new.py +++ b/tests/basics/object_new.py @@ -2,6 +2,14 @@ # (non-initialized) instance of class. # See e.g. http://infohost.nmt.edu/tcc/help/pubs/python/web/new-new-method.html # TODO: Find reference in CPython docs +try: + # If we don't expose object.__new__ (small ports), there's + # nothing to test. + object.__new__ +except AttributeError: + import sys + print("SKIP") + sys.exit() class Foo: diff --git a/tests/basics/op_error.py b/tests/basics/op_error.py index 19ce04bc52..5ba6a80e26 100644 --- a/tests/basics/op_error.py +++ b/tests/basics/op_error.py @@ -20,7 +20,6 @@ test_exc("False in True", TypeError) test_exc("1 * {}", TypeError) test_exc("1 in 1", TypeError) test_exc("bytearray() // 2", TypeError) -test_exc("m = memoryview(bytearray())\nm += bytearray()", TypeError) # object with buffer protocol needed on rhs test_exc("bytearray(1) + 1", TypeError) diff --git a/tests/basics/op_error_memoryview.py b/tests/basics/op_error_memoryview.py new file mode 100644 index 0000000000..658ededc80 --- /dev/null +++ b/tests/basics/op_error_memoryview.py @@ -0,0 +1,19 @@ +# test errors from bad operations (unary, binary, etc) +try: + memoryview +except: + import sys + print("SKIP") + sys.exit() + +def test_exc(code, exc): + try: + exec(code) + print("no exception") + except exc: + print("right exception") + except: + print("wrong exception") + +# unsupported binary operators +test_exc("m = memoryview(bytearray())\nm += bytearray()", TypeError) diff --git a/tests/basics/set_iter_of_iter.py b/tests/basics/set_iter_of_iter.py new file mode 100644 index 0000000000..e3e91fa456 --- /dev/null +++ b/tests/basics/set_iter_of_iter.py @@ -0,0 +1,2 @@ +i = iter(iter({1, 2, 3})) +print(sorted(i)) diff --git a/tests/basics/special_methods.py b/tests/basics/special_methods.py index 1df7a7c4c7..9f57247c12 100644 --- a/tests/basics/special_methods.py +++ b/tests/basics/special_methods.py @@ -105,42 +105,4 @@ cud1 > cud2 cud1 + cud2 cud1 - cud2 -# the following require MICROPY_PY_ALL_SPECIAL_METHODS -+cud1 --cud1 -~cud1 -cud1 * cud2 -cud1 / cud2 -cud2 // cud1 -cud1 += cud2 -cud1 -= cud2 - -# TODO: the following operations are not supported on every ports -# -# ne is not supported, !(eq) is called instead -#cud1 != cud2 -# -# binary and is not supported -# cud1 & cud2 -# -# binary lshift is not supported -# cud1<<1 -# -# modulus is not supported -# cud1 % 2 -# -# binary or is not supported -# cud1 | cud2 -# -# pow is not supported -# cud1**2 -# -# rshift is not suported -# cud1>>1 -# -# xor is not supported -# cud1^cud2 -# -# in the followin test, cpython still calls __eq__ -# cud3=cud1 -# cud3==cud1 +# more in special_methods2.py diff --git a/tests/basics/special_methods2.py b/tests/basics/special_methods2.py new file mode 100644 index 0000000000..3623b30dcc --- /dev/null +++ b/tests/basics/special_methods2.py @@ -0,0 +1,145 @@ +class Cud(): + + def __init__(self): + #print("__init__ called") + pass + + def __repr__(self): + print("__repr__ called") + return "" + + def __lt__(self, other): + print("__lt__ called") + + def __le__(self, other): + print("__le__ called") + + def __eq__(self, other): + print("__eq__ called") + + def __ne__(self, other): + print("__ne__ called") + + def __ge__(self, other): + print("__ge__ called") + + def __gt__(self, other): + print("__gt__ called") + + def __abs__(self): + print("__abs__ called") + + def __add__(self, other): + print("__add__ called") + + def __and__(self, other): + print("__and__ called") + + def __floordiv__(self, other): + print("__floordiv__ called") + + def __index__(self, other): + print("__index__ called") + + def __inv__(self): + print("__inv__ called") + + def __invert__(self): + print("__invert__ called") + + def __lshift__(self, val): + print("__lshift__ called") + + def __mod__(self, val): + print("__mod__ called") + + def __mul__(self, other): + print("__mul__ called") + + def __matmul__(self, other): + print("__matmul__ called") + + def __neg__(self): + print("__neg__ called") + + def __or__(self, other): + print("__or__ called") + + def __pos__(self): + print("__pos__ called") + + def __pow__(self, val): + print("__pow__ called") + + def __rshift__(self, val): + print("__rshift__ called") + + def __sub__(self, other): + print("__sub__ called") + + def __truediv__(self, other): + print("__truediv__ called") + + def __div__(self, other): + print("__div__ called") + + def __xor__(self, other): + print("__xor__ called") + + def __iadd__(self, other): + print("__iadd__ called") + return self + + def __isub__(self, other): + print("__isub__ called") + return self + +cud1 = Cud() +cud2 = Cud() + +try: + +cud1 +except TypeError: + import sys + print("SKIP") + sys.exit() + +# the following require MICROPY_PY_ALL_SPECIAL_METHODS ++cud1 +-cud1 +~cud1 +cud1 * cud2 +cud1 / cud2 +cud2 // cud1 +cud1 += cud2 +cud1 -= cud2 + +# TODO: the following operations are not supported on every ports +# +# ne is not supported, !(eq) is called instead +#cud1 != cud2 +# +# binary and is not supported +# cud1 & cud2 +# +# binary lshift is not supported +# cud1<<1 +# +# modulus is not supported +# cud1 % 2 +# +# binary or is not supported +# cud1 | cud2 +# +# pow is not supported +# cud1**2 +# +# rshift is not suported +# cud1>>1 +# +# xor is not supported +# cud1^cud2 +# +# in the followin test, cpython still calls __eq__ +# cud3=cud1 +# cud3==cud1 diff --git a/tests/basics/subclass_classmethod.py b/tests/basics/subclass_classmethod.py index ae5fbd1aa3..48f164b364 100644 --- a/tests/basics/subclass_classmethod.py +++ b/tests/basics/subclass_classmethod.py @@ -5,6 +5,13 @@ class Base: def foo(cls): print(cls.__name__) +try: + Base.__name__ +except AttributeError: + import sys + print("SKIP") + sys.exit() + class Sub(Base): pass diff --git a/tests/basics/sys1.py b/tests/basics/sys1.py index 816c8823aa..29ef974d14 100644 --- a/tests/basics/sys1.py +++ b/tests/basics/sys1.py @@ -6,8 +6,18 @@ print(sys.__name__) print(type(sys.path)) print(type(sys.argv)) print(sys.byteorder in ('little', 'big')) -print(sys.maxsize > 100) -print(sys.implementation.name in ('cpython', 'micropython')) + +try: + print(sys.maxsize > 100) +except AttributeError: + # Effectively skip subtests + print(True) + +try: + print(sys.implementation.name in ('cpython', 'micropython')) +except AttributeError: + # Effectively skip subtests + print(True) try: sys.exit() From a937750cebc104296f9342e559626ff354772fda Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Wed, 15 Feb 2017 19:20:46 +0300 Subject: [PATCH 42/92] extmod/modlwip: Add my copyright. Per: $ git log modlwip.c |grep ^Auth | sort | uniq -c 9 Author: Damien George 2 Author: Galen Hazelwood 43 Author: Paul Sokolovsky --- extmod/modlwip.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extmod/modlwip.c b/extmod/modlwip.c index 0f2a6c64b4..5b03d7c294 100644 --- a/extmod/modlwip.c +++ b/extmod/modlwip.c @@ -1,10 +1,11 @@ /* - * This file is part of the Micro Python project, http://micropython.org/ + * This file is part of the MicroPython project, http://micropython.org/ * * The MIT License (MIT) * * Copyright (c) 2013, 2014 Damien P. George * Copyright (c) 2015 Galen Hazelwood + * Copyright (c) 2015-2016 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From 4e3bac2e42788aa9312a6145aef18da083eac2df Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 15:32:34 +1100 Subject: [PATCH 43/92] py/runtime: Convert mp_uint_t to size_t where appropriate. --- py/nativeglue.c | 2 +- py/runtime.c | 30 +++++++++++++++--------------- py/runtime.h | 16 ++++++++-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/py/nativeglue.c b/py/nativeglue.c index 5f2164ee0d..5db63080b4 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -86,7 +86,7 @@ mp_obj_t mp_convert_native_to_obj(mp_uint_t val, mp_uint_t type) { // wrapper that accepts n_args and n_kw in one argument // (native emitter can only pass at most 3 arguments to a function) -mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, mp_uint_t n_args_kw, const mp_obj_t *args) { +mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args) { return mp_call_function_n_kw(fun_in, n_args_kw & 0xff, (n_args_kw >> 8) & 0xff, args); } diff --git a/py/runtime.c b/py/runtime.c index e6aef21d77..3ac30f58e3 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -301,7 +301,7 @@ mp_obj_t mp_binary_op(mp_uint_t op, mp_obj_t lhs, mp_obj_t rhs) { } } else if (MP_OBJ_IS_TYPE(rhs, &mp_type_tuple)) { mp_obj_tuple_t *tuple = MP_OBJ_TO_PTR(rhs); - for (mp_uint_t i = 0; i < tuple->len; i++) { + for (size_t i = 0; i < tuple->len; i++) { rhs = tuple->items[i]; if (!mp_obj_is_exception_type(rhs)) { goto unsupported_op; @@ -580,7 +580,7 @@ mp_obj_t mp_call_function_2(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2) { } // args contains, eg: arg0 arg1 key0 value0 key1 value1 -mp_obj_t mp_call_function_n_kw(mp_obj_t fun_in, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args) { +mp_obj_t mp_call_function_n_kw(mp_obj_t fun_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { // TODO improve this: fun object can specify its type and we parse here the arguments, // passing to the function arrays of fixed and keyword arguments @@ -604,7 +604,7 @@ mp_obj_t mp_call_function_n_kw(mp_obj_t fun_in, mp_uint_t n_args, mp_uint_t n_kw // args contains: fun self/NULL arg(0) ... arg(n_args-2) arg(n_args-1) kw_key(0) kw_val(0) ... kw_key(n_kw-1) kw_val(n_kw-1) // if n_args==0 and n_kw==0 then there are only fun and self/NULL -mp_obj_t mp_call_method_n_kw(mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args) { +mp_obj_t mp_call_method_n_kw(size_t n_args, size_t n_kw, const mp_obj_t *args) { DEBUG_OP_printf("call method (fun=%p, self=%p, n_args=" UINT_FMT ", n_kw=" UINT_FMT ", args=%p)\n", args[0], args[1], n_args, n_kw, args); int adjust = (args[1] == MP_OBJ_NULL) ? 0 : 1; return mp_call_function_n_kw(args[0], n_args + adjust, n_kw, args + 2 - adjust); @@ -614,7 +614,7 @@ mp_obj_t mp_call_method_n_kw(mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *a #if !MICROPY_STACKLESS STATIC #endif -void mp_call_prepare_args_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args, mp_call_args_t *out_args) { +void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_obj_t *args, mp_call_args_t *out_args) { mp_obj_t fun = *args++; mp_obj_t self = MP_OBJ_NULL; if (have_self) { @@ -724,7 +724,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const // dictionary mp_map_t *map = mp_obj_dict_get_map(kw_dict); assert(args2_len + 2 * map->used <= args2_alloc); // should have enough, since kw_dict_len is in this case hinted correctly above - for (mp_uint_t i = 0; i < map->alloc; i++) { + for (size_t i = 0; i < map->alloc; i++) { if (MP_MAP_SLOT_IS_FILLED(map, i)) { // the key must be a qstr, so intern it if it's a string mp_obj_t key = map->table[i].key; @@ -780,7 +780,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const out_args->n_alloc = args2_alloc; } -mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args) { +mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_obj_t *args) { mp_call_args_t out_args; mp_call_prepare_args_n_kw_var(have_self, n_args_n_kw, args, &out_args); @@ -791,7 +791,7 @@ mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp } // unpacked items are stored in reverse order into the array pointed to by items -void mp_unpack_sequence(mp_obj_t seq_in, mp_uint_t num, mp_obj_t *items) { +void mp_unpack_sequence(mp_obj_t seq_in, size_t num, mp_obj_t *items) { mp_uint_t seq_len; if (MP_OBJ_IS_TYPE(seq_in, &mp_type_tuple) || MP_OBJ_IS_TYPE(seq_in, &mp_type_list)) { mp_obj_t *seq_items; @@ -805,7 +805,7 @@ void mp_unpack_sequence(mp_obj_t seq_in, mp_uint_t num, mp_obj_t *items) { } else if (seq_len > num) { goto too_long; } - for (mp_uint_t i = 0; i < num; i++) { + for (size_t i = 0; i < num; i++) { items[i] = seq_items[num - 1 - i]; } } else { @@ -841,9 +841,9 @@ too_long: } // unpacked items are stored in reverse order into the array pointed to by items -void mp_unpack_ex(mp_obj_t seq_in, mp_uint_t num_in, mp_obj_t *items) { - mp_uint_t num_left = num_in & 0xff; - mp_uint_t num_right = (num_in >> 8) & 0xff; +void mp_unpack_ex(mp_obj_t seq_in, size_t num_in, mp_obj_t *items) { + size_t num_left = num_in & 0xff; + size_t num_right = (num_in >> 8) & 0xff; DEBUG_OP_printf("unpack ex " UINT_FMT " " UINT_FMT "\n", num_left, num_right); mp_uint_t seq_len; if (MP_OBJ_IS_TYPE(seq_in, &mp_type_tuple) || MP_OBJ_IS_TYPE(seq_in, &mp_type_list)) { @@ -861,11 +861,11 @@ void mp_unpack_ex(mp_obj_t seq_in, mp_uint_t num_in, mp_obj_t *items) { if (seq_len < num_left + num_right) { goto too_short; } - for (mp_uint_t i = 0; i < num_right; i++) { + for (size_t i = 0; i < num_right; i++) { items[i] = seq_items[seq_len - 1 - i]; } items[num_right] = mp_obj_new_list(seq_len - num_left - num_right, seq_items + num_left); - for (mp_uint_t i = 0; i < num_left; i++) { + for (size_t i = 0; i < num_left; i++) { items[num_right + 1 + i] = seq_items[num_left - 1 - i]; } } else { @@ -890,7 +890,7 @@ void mp_unpack_ex(mp_obj_t seq_in, mp_uint_t num_in, mp_obj_t *items) { goto too_short; } items[num_right] = MP_OBJ_FROM_PTR(rest); - for (mp_uint_t i = 0; i < num_right; i++) { + for (size_t i = 0; i < num_right; i++) { items[num_right - 1 - i] = rest->items[rest->len - num_right + i]; } mp_obj_list_set_len(MP_OBJ_FROM_PTR(rest), rest->len - num_right); @@ -1350,7 +1350,7 @@ void mp_import_all(mp_obj_t module) { // TODO: Support __all__ mp_map_t *map = mp_obj_dict_get_map(MP_OBJ_FROM_PTR(mp_obj_module_get_globals(module))); - for (mp_uint_t i = 0; i < map->alloc; i++) { + for (size_t i = 0; i < map->alloc; i++) { if (MP_MAP_SLOT_IS_FILLED(map, i)) { qstr name = MP_OBJ_QSTR_VALUE(map->table[i].key); if (*qstr_str(name) != '_') { diff --git a/py/runtime.h b/py/runtime.h index 3532b838de..e25f2a483d 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -92,9 +92,9 @@ mp_obj_t mp_binary_op(mp_uint_t op, mp_obj_t lhs, mp_obj_t rhs); mp_obj_t mp_call_function_0(mp_obj_t fun); mp_obj_t mp_call_function_1(mp_obj_t fun, mp_obj_t arg); mp_obj_t mp_call_function_2(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2); -mp_obj_t mp_call_function_n_kw(mp_obj_t fun, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args); -mp_obj_t mp_call_method_n_kw(mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args); -mp_obj_t mp_call_method_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args); +mp_obj_t mp_call_function_n_kw(mp_obj_t fun, size_t n_args, size_t n_kw, const mp_obj_t *args); +mp_obj_t mp_call_method_n_kw(size_t n_args, size_t n_kw, const mp_obj_t *args); +mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_obj_t *args); mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, size_t n_kw, const mp_obj_t *args); // Call function and catch/dump exception - for Python callbacks from C code void mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg); @@ -102,7 +102,7 @@ void mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2); typedef struct _mp_call_args_t { mp_obj_t fun; - mp_uint_t n_args, n_kw, n_alloc; + size_t n_args, n_kw, n_alloc; mp_obj_t *args; } mp_call_args_t; @@ -111,11 +111,11 @@ typedef struct _mp_call_args_t { // prepares argument array suitable for passing to ->call() method of a // function object (and mp_call_function_n_kw()). // (Only needed in stackless mode.) -void mp_call_prepare_args_n_kw_var(bool have_self, mp_uint_t n_args_n_kw, const mp_obj_t *args, mp_call_args_t *out_args); +void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_obj_t *args, mp_call_args_t *out_args); #endif -void mp_unpack_sequence(mp_obj_t seq, mp_uint_t num, mp_obj_t *items); -void mp_unpack_ex(mp_obj_t seq, mp_uint_t num, mp_obj_t *items); +void mp_unpack_sequence(mp_obj_t seq, size_t num, mp_obj_t *items); +void mp_unpack_ex(mp_obj_t seq, size_t num, mp_obj_t *items); mp_obj_t mp_store_map(mp_obj_t map, mp_obj_t key, mp_obj_t value); mp_obj_t mp_load_attr(mp_obj_t base, qstr attr); void mp_convert_member_lookup(mp_obj_t obj, const mp_obj_type_t *type, mp_obj_t member, mp_obj_t *dest); @@ -155,7 +155,7 @@ NORETURN void mp_exc_recursion_depth(void); // helper functions for native/viper code mp_uint_t mp_convert_obj_to_native(mp_obj_t obj, mp_uint_t type); mp_obj_t mp_convert_native_to_obj(mp_uint_t val, mp_uint_t type); -mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, mp_uint_t n_args_kw, const mp_obj_t *args); +mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args); void mp_native_raise(mp_obj_t o); #define mp_sys_path (MP_OBJ_FROM_PTR(&MP_STATE_VM(mp_sys_path_obj))) From dcdcc43dad0e1124a69e20fb6d008c7ffaf0e42a Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 15:50:28 +1100 Subject: [PATCH 44/92] py/mpz: Convert mp_uint_t to size_t where appropriate. --- py/mpz.c | 60 ++++++++++++++++++++++++++++---------------------------- py/mpz.h | 20 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/py/mpz.c b/py/mpz.c index 230eb921c6..7f2b257673 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -53,7 +53,7 @@ returns sign(i - j) assumes i, j are normalised */ -STATIC int mpn_cmp(const mpz_dig_t *idig, mp_uint_t ilen, const mpz_dig_t *jdig, mp_uint_t jlen) { +STATIC int mpn_cmp(const mpz_dig_t *idig, size_t ilen, const mpz_dig_t *jdig, size_t jlen) { if (ilen < jlen) { return -1; } if (ilen > jlen) { return 1; } @@ -71,7 +71,7 @@ STATIC int mpn_cmp(const mpz_dig_t *idig, mp_uint_t ilen, const mpz_dig_t *jdig, assumes enough memory in i; assumes normalised j; assumes n > 0 can have i, j pointing to same memory */ -STATIC mp_uint_t mpn_shl(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mp_uint_t n) { +STATIC size_t mpn_shl(mpz_dig_t *idig, mpz_dig_t *jdig, size_t jlen, mp_uint_t n) { mp_uint_t n_whole = (n + DIG_SIZE - 1) / DIG_SIZE; mp_uint_t n_part = n % DIG_SIZE; if (n_part == 0) { @@ -84,7 +84,7 @@ STATIC mp_uint_t mpn_shl(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mp_ui // shift the digits mpz_dbl_dig_t d = 0; - for (mp_uint_t i = jlen; i > 0; i--, idig--, jdig--) { + for (size_t i = jlen; i > 0; i--, idig--, jdig--) { d |= *jdig; *idig = (d >> (DIG_SIZE - n_part)) & DIG_MASK; d <<= DIG_SIZE; @@ -110,7 +110,7 @@ STATIC mp_uint_t mpn_shl(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mp_ui assumes enough memory in i; assumes normalised j; assumes n > 0 can have i, j pointing to same memory */ -STATIC mp_uint_t mpn_shr(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mp_uint_t n) { +STATIC size_t mpn_shr(mpz_dig_t *idig, mpz_dig_t *jdig, size_t jlen, mp_uint_t n) { mp_uint_t n_whole = n / DIG_SIZE; mp_uint_t n_part = n % DIG_SIZE; @@ -121,7 +121,7 @@ STATIC mp_uint_t mpn_shr(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mp_ui jdig += n_whole; jlen -= n_whole; - for (mp_uint_t i = jlen; i > 0; i--, idig++, jdig++) { + for (size_t i = jlen; i > 0; i--, idig++, jdig++) { mpz_dbl_dig_t d = *jdig; if (i > 1) { d |= (mpz_dbl_dig_t)jdig[1] << DIG_SIZE; @@ -142,7 +142,7 @@ STATIC mp_uint_t mpn_shr(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mp_ui assumes enough memory in i; assumes normalised j, k; assumes jlen >= klen can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_add(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen) { +STATIC size_t mpn_add(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen) { mpz_dig_t *oidig = idig; mpz_dbl_dig_t carry = 0; @@ -172,7 +172,7 @@ STATIC mp_uint_t mpn_add(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, assumes enough memory in i; assumes normalised j, k; assumes j >= k can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_sub(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen) { +STATIC size_t mpn_sub(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen) { mpz_dig_t *oidig = idig; mpz_dbl_dig_signed_t borrow = 0; @@ -196,7 +196,7 @@ STATIC mp_uint_t mpn_sub(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, return idig + 1 - oidig; } -STATIC mp_uint_t mpn_remove_trailing_zeros(mpz_dig_t *oidig, mpz_dig_t *idig) { +STATIC size_t mpn_remove_trailing_zeros(mpz_dig_t *oidig, mpz_dig_t *idig) { for (--idig; idig >= oidig && *idig == 0; --idig) { } return idig + 1 - oidig; @@ -209,7 +209,7 @@ STATIC mp_uint_t mpn_remove_trailing_zeros(mpz_dig_t *oidig, mpz_dig_t *idig) { assumes enough memory in i; assumes normalised j, k; assumes jlen >= klen (jlen argument not needed) can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_and(mpz_dig_t *idig, const mpz_dig_t *jdig, const mpz_dig_t *kdig, mp_uint_t klen) { +STATIC size_t mpn_and(mpz_dig_t *idig, const mpz_dig_t *jdig, const mpz_dig_t *kdig, size_t klen) { mpz_dig_t *oidig = idig; for (; klen > 0; --klen, ++idig, ++jdig, ++kdig) { @@ -230,7 +230,7 @@ STATIC mp_uint_t mpn_and(mpz_dig_t *idig, const mpz_dig_t *jdig, const mpz_dig_t assumes enough memory in i; assumes normalised j, k; assumes length j >= length k can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_and_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen, +STATIC size_t mpn_and_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen, mpz_dbl_dig_t carryi, mpz_dbl_dig_t carryj, mpz_dbl_dig_t carryk) { mpz_dig_t *oidig = idig; mpz_dig_t imask = (0 == carryi) ? 0 : DIG_MASK; @@ -261,7 +261,7 @@ STATIC mp_uint_t mpn_and_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t j assumes enough memory in i; assumes normalised j, k; assumes jlen >= klen can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_or(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen) { +STATIC size_t mpn_or(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen) { mpz_dig_t *oidig = idig; jlen -= klen; @@ -291,7 +291,7 @@ STATIC mp_uint_t mpn_or(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, #if MICROPY_OPT_MPZ_BITWISE -STATIC mp_uint_t mpn_or_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen, +STATIC size_t mpn_or_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen, mpz_dbl_dig_t carryj, mpz_dbl_dig_t carryk) { mpz_dig_t *oidig = idig; mpz_dbl_dig_t carryi = 1; @@ -321,7 +321,7 @@ STATIC mp_uint_t mpn_or_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jl #else -STATIC mp_uint_t mpn_or_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen, +STATIC size_t mpn_or_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen, mpz_dbl_dig_t carryi, mpz_dbl_dig_t carryj, mpz_dbl_dig_t carryk) { mpz_dig_t *oidig = idig; mpz_dig_t imask = (0 == carryi) ? 0 : DIG_MASK; @@ -353,7 +353,7 @@ STATIC mp_uint_t mpn_or_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jl assumes enough memory in i; assumes normalised j, k; assumes jlen >= klen can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_xor(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen) { +STATIC size_t mpn_xor(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen) { mpz_dig_t *oidig = idig; jlen -= klen; @@ -380,7 +380,7 @@ STATIC mp_uint_t mpn_xor(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, assumes enough memory in i; assumes normalised j, k; assumes length j >= length k can have i, j, k pointing to same memory */ -STATIC mp_uint_t mpn_xor_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t jlen, const mpz_dig_t *kdig, mp_uint_t klen, +STATIC size_t mpn_xor_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, size_t jlen, const mpz_dig_t *kdig, size_t klen, mpz_dbl_dig_t carryi, mpz_dbl_dig_t carryj, mpz_dbl_dig_t carryk) { mpz_dig_t *oidig = idig; @@ -405,7 +405,7 @@ STATIC mp_uint_t mpn_xor_neg(mpz_dig_t *idig, const mpz_dig_t *jdig, mp_uint_t j returns number of digits in i assumes enough memory in i; assumes normalised i; assumes dmul != 0 */ -STATIC mp_uint_t mpn_mul_dig_add_dig(mpz_dig_t *idig, mp_uint_t ilen, mpz_dig_t dmul, mpz_dig_t dadd) { +STATIC size_t mpn_mul_dig_add_dig(mpz_dig_t *idig, size_t ilen, mpz_dig_t dmul, mpz_dig_t dadd) { mpz_dig_t *oidig = idig; mpz_dbl_dig_t carry = dadd; @@ -427,15 +427,15 @@ STATIC mp_uint_t mpn_mul_dig_add_dig(mpz_dig_t *idig, mp_uint_t ilen, mpz_dig_t assumes enough memory in i; assumes i is zeroed; assumes normalised j, k can have j, k point to same memory */ -STATIC mp_uint_t mpn_mul(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mpz_dig_t *kdig, mp_uint_t klen) { +STATIC size_t mpn_mul(mpz_dig_t *idig, mpz_dig_t *jdig, size_t jlen, mpz_dig_t *kdig, size_t klen) { mpz_dig_t *oidig = idig; - mp_uint_t ilen = 0; + size_t ilen = 0; for (; klen > 0; --klen, ++idig, ++kdig) { mpz_dig_t *id = idig; mpz_dbl_dig_t carry = 0; - mp_uint_t jl = jlen; + size_t jl = jlen; for (mpz_dig_t *jd = jdig; jl > 0; --jl, ++jd, ++id) { carry += (mpz_dbl_dig_t)*id + (mpz_dbl_dig_t)*jd * (mpz_dbl_dig_t)*kdig; // will never overflow so long as DIG_SIZE <= 8*sizeof(mpz_dbl_dig_t)/2 *id = carry & DIG_MASK; @@ -458,7 +458,7 @@ STATIC mp_uint_t mpn_mul(mpz_dig_t *idig, mpz_dig_t *jdig, mp_uint_t jlen, mpz_d assumes quo_dig has enough memory (as many digits as num) assumes quo_dig is filled with zeros */ -STATIC void mpn_div(mpz_dig_t *num_dig, mp_uint_t *num_len, const mpz_dig_t *den_dig, mp_uint_t den_len, mpz_dig_t *quo_dig, mp_uint_t *quo_len) { +STATIC void mpn_div(mpz_dig_t *num_dig, size_t *num_len, const mpz_dig_t *den_dig, size_t den_len, mpz_dig_t *quo_dig, size_t *quo_len) { mpz_dig_t *orig_num_dig = num_dig; mpz_dig_t *orig_quo_dig = quo_dig; mpz_dig_t norm_shift = 0; @@ -661,7 +661,7 @@ void mpz_init_from_int(mpz_t *z, mp_int_t val) { mpz_set_from_int(z, val); } -void mpz_init_fixed_from_int(mpz_t *z, mpz_dig_t *dig, mp_uint_t alloc, mp_int_t val) { +void mpz_init_fixed_from_int(mpz_t *z, mpz_dig_t *dig, size_t alloc, mp_int_t val) { z->neg = 0; z->fixed_dig = 1; z->alloc = alloc; @@ -705,7 +705,7 @@ mpz_t *mpz_from_float(mp_float_t val) { } #endif -mpz_t *mpz_from_str(const char *str, mp_uint_t len, bool neg, mp_uint_t base) { +mpz_t *mpz_from_str(const char *str, size_t len, bool neg, mp_uint_t base) { mpz_t *z = mpz_zero(); mpz_set_from_str(z, str, len, neg, base); return z; @@ -719,7 +719,7 @@ STATIC void mpz_free(mpz_t *z) { } } -STATIC void mpz_need_dig(mpz_t *z, mp_uint_t need) { +STATIC void mpz_need_dig(mpz_t *z, size_t need) { if (need < MIN_ALLOC) { need = MIN_ALLOC; } @@ -873,7 +873,7 @@ typedef uint32_t mp_float_int_t; #endif // returns number of bytes from str that were processed -mp_uint_t mpz_set_from_str(mpz_t *z, const char *str, mp_uint_t len, bool neg, mp_uint_t base) { +size_t mpz_set_from_str(mpz_t *z, const char *str, size_t len, bool neg, mp_uint_t base) { assert(base <= 36); const char *cur = str; @@ -909,7 +909,7 @@ mp_uint_t mpz_set_from_str(mpz_t *z, const char *str, mp_uint_t len, bool neg, m return cur - str; } -void mpz_set_from_bytes(mpz_t *z, bool big_endian, mp_uint_t len, const byte *buf) { +void mpz_set_from_bytes(mpz_t *z, bool big_endian, size_t len, const byte *buf) { int delta = 1; if (big_endian) { buf += len - 1; @@ -1151,7 +1151,7 @@ void mpz_shr_inpl(mpz_t *dest, const mpz_t *lhs, mp_uint_t rhs) { mp_uint_t n_whole = rhs / DIG_SIZE; mp_uint_t n_part = rhs % DIG_SIZE; mpz_dig_t round_up = 0; - for (mp_uint_t i = 0; i < lhs->len && i < n_whole; i++) { + for (size_t i = 0; i < lhs->len && i < n_whole; i++) { if (lhs->dig[i] != 0) { round_up = 1; break; @@ -1624,7 +1624,7 @@ bool mpz_as_uint_checked(const mpz_t *i, mp_uint_t *value) { } // writes at most len bytes to buf (so buf should be zeroed before calling) -void mpz_as_bytes(const mpz_t *z, bool big_endian, mp_uint_t len, byte *buf) { +void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf) { byte *b = buf; if (big_endian) { b += len; @@ -1633,7 +1633,7 @@ void mpz_as_bytes(const mpz_t *z, bool big_endian, mp_uint_t len, byte *buf) { int bits = 0; mpz_dbl_dig_t d = 0; mpz_dbl_dig_t carry = 1; - for (mp_uint_t zlen = z->len; zlen > 0; --zlen) { + for (size_t zlen = z->len; zlen > 0; --zlen) { bits += DIG_SIZE; d = (d << DIG_SIZE) | *zdig++; for (; bits >= 8; bits -= 8, d >>= 8) { @@ -1685,7 +1685,7 @@ char *mpz_as_str(const mpz_t *i, mp_uint_t base) { // assumes enough space as calculated by mp_int_format_size // returns length of string, not including null byte -mp_uint_t mpz_as_str_inpl(const mpz_t *i, mp_uint_t base, const char *prefix, char base_char, char comma, char *str) { +size_t mpz_as_str_inpl(const mpz_t *i, mp_uint_t base, const char *prefix, char base_char, char comma, char *str) { if (str == NULL) { return 0; } @@ -1694,7 +1694,7 @@ mp_uint_t mpz_as_str_inpl(const mpz_t *i, mp_uint_t base, const char *prefix, ch return 0; } - mp_uint_t ilen = i->len; + size_t ilen = i->len; char *s = str; if (ilen == 0) { diff --git a/py/mpz.h b/py/mpz.h index 8facb1a0f3..c792d87c56 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -87,10 +87,10 @@ typedef int8_t mpz_dbl_dig_signed_t; #define MPZ_NUM_DIG_FOR_LL ((sizeof(long long) * 8 + MPZ_DIG_SIZE - 1) / MPZ_DIG_SIZE) typedef struct _mpz_t { - mp_uint_t neg : 1; - mp_uint_t fixed_dig : 1; - mp_uint_t alloc : BITS_PER_WORD - 2; - mp_uint_t len; + size_t neg : 1; + size_t fixed_dig : 1; + size_t alloc : 8 * sizeof(size_t) - 2; + size_t len; mpz_dig_t *dig; } mpz_t; @@ -99,7 +99,7 @@ typedef struct _mpz_t { void mpz_init_zero(mpz_t *z); void mpz_init_from_int(mpz_t *z, mp_int_t val); -void mpz_init_fixed_from_int(mpz_t *z, mpz_dig_t *dig, mp_uint_t dig_alloc, mp_int_t val); +void mpz_init_fixed_from_int(mpz_t *z, mpz_dig_t *dig, size_t dig_alloc, mp_int_t val); void mpz_deinit(mpz_t *z); void mpz_set(mpz_t *dest, const mpz_t *src); @@ -108,8 +108,8 @@ void mpz_set_from_ll(mpz_t *z, long long i, bool is_signed); #if MICROPY_PY_BUILTINS_FLOAT void mpz_set_from_float(mpz_t *z, mp_float_t src); #endif -mp_uint_t mpz_set_from_str(mpz_t *z, const char *str, mp_uint_t len, bool neg, mp_uint_t base); -void mpz_set_from_bytes(mpz_t *z, bool big_endian, mp_uint_t len, const byte *buf); +size_t mpz_set_from_str(mpz_t *z, const char *str, size_t len, bool neg, mp_uint_t base); +void mpz_set_from_bytes(mpz_t *z, bool big_endian, size_t len, const byte *buf); bool mpz_is_zero(const mpz_t *z); int mpz_cmp(const mpz_t *lhs, const mpz_t *rhs); @@ -133,11 +133,11 @@ static inline size_t mpz_max_num_bits(const mpz_t *z) { return z->len * MPZ_DIG_ mp_int_t mpz_hash(const mpz_t *z); bool mpz_as_int_checked(const mpz_t *z, mp_int_t *value); bool mpz_as_uint_checked(const mpz_t *z, mp_uint_t *value); -void mpz_as_bytes(const mpz_t *z, bool big_endian, mp_uint_t len, byte *buf); +void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf); #if MICROPY_PY_BUILTINS_FLOAT mp_float_t mpz_as_float(const mpz_t *z); #endif -mp_uint_t mpz_as_str_size(const mpz_t *i, mp_uint_t base, const char *prefix, char comma); -mp_uint_t mpz_as_str_inpl(const mpz_t *z, mp_uint_t base, const char *prefix, char base_char, char comma, char *str); +size_t mpz_as_str_size(const mpz_t *i, mp_uint_t base, const char *prefix, char comma); +size_t mpz_as_str_inpl(const mpz_t *z, mp_uint_t base, const char *prefix, char base_char, char comma, char *str); #endif // __MICROPY_INCLUDED_PY_MPZ_H__ From eb90edb5c05dd613c144e2b66dd303f2248593e5 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 15:55:36 +1100 Subject: [PATCH 45/92] py/mpz: Remove obsolete declaration of mpz_as_str_size. --- py/mpz.c | 2 +- py/mpz.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/py/mpz.c b/py/mpz.c index 7f2b257673..1090d58c76 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -1677,7 +1677,7 @@ mp_float_t mpz_as_float(const mpz_t *i) { #if 0 this function is unused char *mpz_as_str(const mpz_t *i, mp_uint_t base) { - char *s = m_new(char, mpz_as_str_size(i, base, NULL, '\0')); + char *s = m_new(char, mp_int_format_size(mpz_max_num_bits(i), base, NULL, '\0')); mpz_as_str_inpl(i, base, NULL, 'a', '\0', s); return s; } diff --git a/py/mpz.h b/py/mpz.h index c792d87c56..1e91ca3304 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -137,7 +137,6 @@ void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf); #if MICROPY_PY_BUILTINS_FLOAT mp_float_t mpz_as_float(const mpz_t *z); #endif -size_t mpz_as_str_size(const mpz_t *i, mp_uint_t base, const char *prefix, char comma); size_t mpz_as_str_inpl(const mpz_t *z, mp_uint_t base, const char *prefix, char base_char, char comma, char *str); #endif // __MICROPY_INCLUDED_PY_MPZ_H__ From 6ed77bedbded21f4f3513b21fbb8b17a1f3e709e Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 15:59:51 +1100 Subject: [PATCH 46/92] py/mpz: Change type of "base" args from mp_uint_t to unsigned int. --- py/mpz.c | 8 ++++---- py/mpz.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/py/mpz.c b/py/mpz.c index 1090d58c76..72d226cb3f 100644 --- a/py/mpz.c +++ b/py/mpz.c @@ -705,7 +705,7 @@ mpz_t *mpz_from_float(mp_float_t val) { } #endif -mpz_t *mpz_from_str(const char *str, size_t len, bool neg, mp_uint_t base) { +mpz_t *mpz_from_str(const char *str, size_t len, bool neg, unsigned int base) { mpz_t *z = mpz_zero(); mpz_set_from_str(z, str, len, neg, base); return z; @@ -873,7 +873,7 @@ typedef uint32_t mp_float_int_t; #endif // returns number of bytes from str that were processed -size_t mpz_set_from_str(mpz_t *z, const char *str, size_t len, bool neg, mp_uint_t base) { +size_t mpz_set_from_str(mpz_t *z, const char *str, size_t len, bool neg, unsigned int base) { assert(base <= 36); const char *cur = str; @@ -1676,7 +1676,7 @@ mp_float_t mpz_as_float(const mpz_t *i) { #if 0 this function is unused -char *mpz_as_str(const mpz_t *i, mp_uint_t base) { +char *mpz_as_str(const mpz_t *i, unsigned int base) { char *s = m_new(char, mp_int_format_size(mpz_max_num_bits(i), base, NULL, '\0')); mpz_as_str_inpl(i, base, NULL, 'a', '\0', s); return s; @@ -1685,7 +1685,7 @@ char *mpz_as_str(const mpz_t *i, mp_uint_t base) { // assumes enough space as calculated by mp_int_format_size // returns length of string, not including null byte -size_t mpz_as_str_inpl(const mpz_t *i, mp_uint_t base, const char *prefix, char base_char, char comma, char *str) { +size_t mpz_as_str_inpl(const mpz_t *i, unsigned int base, const char *prefix, char base_char, char comma, char *str) { if (str == NULL) { return 0; } diff --git a/py/mpz.h b/py/mpz.h index 1e91ca3304..5c88227223 100644 --- a/py/mpz.h +++ b/py/mpz.h @@ -108,7 +108,7 @@ void mpz_set_from_ll(mpz_t *z, long long i, bool is_signed); #if MICROPY_PY_BUILTINS_FLOAT void mpz_set_from_float(mpz_t *z, mp_float_t src); #endif -size_t mpz_set_from_str(mpz_t *z, const char *str, size_t len, bool neg, mp_uint_t base); +size_t mpz_set_from_str(mpz_t *z, const char *str, size_t len, bool neg, unsigned int base); void mpz_set_from_bytes(mpz_t *z, bool big_endian, size_t len, const byte *buf); bool mpz_is_zero(const mpz_t *z); @@ -137,6 +137,6 @@ void mpz_as_bytes(const mpz_t *z, bool big_endian, size_t len, byte *buf); #if MICROPY_PY_BUILTINS_FLOAT mp_float_t mpz_as_float(const mpz_t *z); #endif -size_t mpz_as_str_inpl(const mpz_t *z, mp_uint_t base, const char *prefix, char base_char, char comma, char *str); +size_t mpz_as_str_inpl(const mpz_t *z, unsigned int base, const char *prefix, char base_char, char comma, char *str); #endif // __MICROPY_INCLUDED_PY_MPZ_H__ From 891dc5c62ce9b266f0352fc96ed78f36ee9fc825 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:04:13 +1100 Subject: [PATCH 47/92] py/persistentcode: Replace mp_uint_t with size_t where appropriate. --- py/persistentcode.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/py/persistentcode.c b/py/persistentcode.c index 99b01f8e27..07395b304b 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -108,8 +108,8 @@ STATIC void read_bytes(mp_reader_t *reader, byte *buf, size_t len) { } } -STATIC mp_uint_t read_uint(mp_reader_t *reader) { - mp_uint_t unum = 0; +STATIC size_t read_uint(mp_reader_t *reader) { + size_t unum = 0; for (;;) { byte b = reader->readbyte(reader->data); unum = (unum << 7) | (b & 0x7f); @@ -121,7 +121,7 @@ STATIC mp_uint_t read_uint(mp_reader_t *reader) { } STATIC qstr load_qstr(mp_reader_t *reader) { - mp_uint_t len = read_uint(reader); + size_t len = read_uint(reader); char *str = m_new(char, len); read_bytes(reader, (byte*)str, len); qstr qst = qstr_from_strn(str, len); @@ -164,7 +164,7 @@ STATIC void load_bytecode_qstrs(mp_reader_t *reader, byte *ip, byte *ip_top) { STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { // load bytecode - mp_uint_t bc_len = read_uint(reader); + size_t bc_len = read_uint(reader); byte *bytecode = m_new(byte, bc_len); read_bytes(reader, bytecode, bc_len); @@ -182,17 +182,17 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { load_bytecode_qstrs(reader, (byte*)ip, bytecode + bc_len); // load constant table - mp_uint_t n_obj = read_uint(reader); - mp_uint_t n_raw_code = read_uint(reader); + size_t n_obj = read_uint(reader); + size_t n_raw_code = read_uint(reader); mp_uint_t *const_table = m_new(mp_uint_t, prelude.n_pos_args + prelude.n_kwonly_args + n_obj + n_raw_code); mp_uint_t *ct = const_table; - for (mp_uint_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { + for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { *ct++ = (mp_uint_t)MP_OBJ_NEW_QSTR(load_qstr(reader)); } - for (mp_uint_t i = 0; i < n_obj; ++i) { + for (size_t i = 0; i < n_obj; ++i) { *ct++ = (mp_uint_t)load_obj(reader); } - for (mp_uint_t i = 0; i < n_raw_code; ++i) { + for (size_t i = 0; i < n_raw_code; ++i) { *ct++ = (mp_uint_t)(uintptr_t)load_raw_code(reader); } @@ -248,7 +248,7 @@ STATIC void mp_print_bytes(mp_print_t *print, const byte *data, size_t len) { } #define BYTES_FOR_INT ((BYTES_PER_WORD * 8 + 6) / 7) -STATIC void mp_print_uint(mp_print_t *print, mp_uint_t n) { +STATIC void mp_print_uint(mp_print_t *print, size_t n) { byte buf[BYTES_FOR_INT]; byte *p = buf + sizeof(buf); *--p = n & 0x7f; From 229823942c79b1233c43dc155bae01c20a8cec57 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:09:51 +1100 Subject: [PATCH 48/92] py/objtuple: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 2 +- py/objattrtuple.c | 10 +++++----- py/objtuple.c | 18 +++++++++--------- py/objtuple.h | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/py/obj.h b/py/obj.h index 071e38502b..95b6ec4119 100644 --- a/py/obj.h +++ b/py/obj.h @@ -630,7 +630,7 @@ mp_obj_t mp_obj_new_fun_viper(mp_uint_t n_args, void *fun_data, mp_uint_t type_s mp_obj_t mp_obj_new_fun_asm(mp_uint_t n_args, void *fun_data, mp_uint_t type_sig); mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed, const mp_obj_t *closed); -mp_obj_t mp_obj_new_tuple(mp_uint_t n, const mp_obj_t *items); +mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); mp_obj_t mp_obj_new_list(mp_uint_t n, mp_obj_t *items); mp_obj_t mp_obj_new_dict(mp_uint_t n_args); mp_obj_t mp_obj_new_set(mp_uint_t n_args, mp_obj_t *items); diff --git a/py/objattrtuple.c b/py/objattrtuple.c index c6dd3aeacf..8c5e795757 100644 --- a/py/objattrtuple.c +++ b/py/objattrtuple.c @@ -34,7 +34,7 @@ STATIC #endif void mp_obj_attrtuple_print_helper(const mp_print_t *print, const qstr *fields, mp_obj_tuple_t *o) { mp_print_str(print, "("); - for (mp_uint_t i = 0; i < o->len; i++) { + for (size_t i = 0; i < o->len; i++) { if (i > 0) { mp_print_str(print, ", "); } @@ -59,9 +59,9 @@ STATIC void mp_obj_attrtuple_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (dest[0] == MP_OBJ_NULL) { // load attribute mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in); - mp_uint_t len = self->len; + size_t len = self->len; const qstr *fields = (const qstr*)MP_OBJ_TO_PTR(self->items[len]); - for (mp_uint_t i = 0; i < len; i++) { + for (size_t i = 0; i < len; i++) { if (fields[i] == attr) { dest[0] = self->items[i]; return; @@ -70,11 +70,11 @@ STATIC void mp_obj_attrtuple_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } } -mp_obj_t mp_obj_new_attrtuple(const qstr *fields, mp_uint_t n, const mp_obj_t *items) { +mp_obj_t mp_obj_new_attrtuple(const qstr *fields, size_t n, const mp_obj_t *items) { mp_obj_tuple_t *o = m_new_obj_var(mp_obj_tuple_t, mp_obj_t, n + 1); o->base.type = &mp_type_attrtuple; o->len = n; - for (mp_uint_t i = 0; i < n; i++) { + for (size_t i = 0; i < n; i++) { o->items[i] = items[i]; } o->items[n] = MP_OBJ_FROM_PTR(fields); diff --git a/py/objtuple.c b/py/objtuple.c index c547da9406..1935a67098 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -32,7 +32,7 @@ #include "py/runtime0.h" #include "py/runtime.h" -STATIC mp_obj_t mp_obj_new_tuple_iterator(mp_obj_tuple_t *tuple, mp_uint_t cur); +STATIC mp_obj_t mp_obj_new_tuple_iterator(mp_obj_tuple_t *tuple, size_t cur); /******************************************************************************/ /* tuple */ @@ -45,7 +45,7 @@ void mp_obj_tuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_print_str(print, "("); kind = PRINT_REPR; } - for (mp_uint_t i = 0; i < o->len; i++) { + for (size_t i = 0; i < o->len; i++) { if (i > 0) { mp_print_str(print, ", "); } @@ -80,8 +80,8 @@ STATIC mp_obj_t mp_obj_tuple_make_new(const mp_obj_type_t *type_in, size_t n_arg // TODO optimise for cases where we know the length of the iterator - mp_uint_t alloc = 4; - mp_uint_t len = 0; + size_t alloc = 4; + size_t len = 0; mp_obj_t *items = m_new(mp_obj_t, alloc); mp_obj_t iterable = mp_getiter(args[0]); @@ -127,7 +127,7 @@ mp_obj_t mp_obj_tuple_unary_op(mp_uint_t op, mp_obj_t self_in) { case MP_UNARY_OP_HASH: { // start hash with pointer to empty tuple, to make it fairly unique mp_int_t hash = (mp_int_t)mp_const_empty_tuple; - for (mp_uint_t i = 0; i < self->len; i++) { + for (size_t i = 0; i < self->len; i++) { hash += MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, self->items[i])); } return MP_OBJ_NEW_SMALL_INT(hash); @@ -235,7 +235,7 @@ const mp_obj_type_t mp_type_tuple = { // the zero-length tuple const mp_obj_tuple_t mp_const_empty_tuple_obj = {{&mp_type_tuple}, 0}; -mp_obj_t mp_obj_new_tuple(mp_uint_t n, const mp_obj_t *items) { +mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items) { if (n == 0) { return mp_const_empty_tuple; } @@ -243,7 +243,7 @@ mp_obj_t mp_obj_new_tuple(mp_uint_t n, const mp_obj_t *items) { o->base.type = &mp_type_tuple; o->len = n; if (items) { - for (mp_uint_t i = 0; i < n; i++) { + for (size_t i = 0; i < n; i++) { o->items[i] = items[i]; } } @@ -270,7 +270,7 @@ typedef struct _mp_obj_tuple_it_t { mp_obj_base_t base; mp_fun_1_t iternext; mp_obj_tuple_t *tuple; - mp_uint_t cur; + size_t cur; } mp_obj_tuple_it_t; STATIC mp_obj_t tuple_it_iternext(mp_obj_t self_in) { @@ -284,7 +284,7 @@ STATIC mp_obj_t tuple_it_iternext(mp_obj_t self_in) { } } -STATIC mp_obj_t mp_obj_new_tuple_iterator(mp_obj_tuple_t *tuple, mp_uint_t cur) { +STATIC mp_obj_t mp_obj_new_tuple_iterator(mp_obj_tuple_t *tuple, size_t cur) { mp_obj_tuple_it_t *o = m_new_obj(mp_obj_tuple_it_t); o->base.type = &mp_type_polymorph_iter; o->iternext = tuple_it_iternext; diff --git a/py/objtuple.h b/py/objtuple.h index ebfc5c4066..760135f866 100644 --- a/py/objtuple.h +++ b/py/objtuple.h @@ -30,13 +30,13 @@ typedef struct _mp_obj_tuple_t { mp_obj_base_t base; - mp_uint_t len; + size_t len; mp_obj_t items[]; } mp_obj_tuple_t; typedef struct _mp_rom_obj_tuple_t { mp_obj_base_t base; - mp_uint_t len; + size_t len; mp_rom_obj_t items[]; } mp_rom_obj_tuple_t; @@ -59,6 +59,6 @@ extern const mp_obj_type_t mp_type_attrtuple; void mp_obj_attrtuple_print_helper(const mp_print_t *print, const qstr *fields, mp_obj_tuple_t *o); #endif -mp_obj_t mp_obj_new_attrtuple(const qstr *fields, mp_uint_t n, const mp_obj_t *items); +mp_obj_t mp_obj_new_attrtuple(const qstr *fields, size_t n, const mp_obj_t *items); #endif // __MICROPY_INCLUDED_PY_OBJTUPLE_H__ From 58d9eeb8d940a6c42f904af70fa01e03c760d832 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:12:41 +1100 Subject: [PATCH 49/92] py/objlist: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 6 +++--- py/objlist.c | 22 +++++++++++----------- py/objlist.h | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/py/obj.h b/py/obj.h index 95b6ec4119..fb35b67aa4 100644 --- a/py/obj.h +++ b/py/obj.h @@ -631,7 +631,7 @@ mp_obj_t mp_obj_new_fun_asm(mp_uint_t n_args, void *fun_data, mp_uint_t type_sig mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed, const mp_obj_t *closed); mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); -mp_obj_t mp_obj_new_list(mp_uint_t n, mp_obj_t *items); +mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items); mp_obj_t mp_obj_new_dict(mp_uint_t n_args); mp_obj_t mp_obj_new_set(mp_uint_t n_args, mp_obj_t *items); mp_obj_t mp_obj_new_slice(mp_obj_t start, mp_obj_t stop, mp_obj_t step); @@ -720,11 +720,11 @@ mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); // list struct _mp_obj_list_t; -void mp_obj_list_init(struct _mp_obj_list_t *o, mp_uint_t n); +void mp_obj_list_init(struct _mp_obj_list_t *o, size_t n); mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); void mp_obj_list_get(mp_obj_t self_in, mp_uint_t *len, mp_obj_t **items); -void mp_obj_list_set_len(mp_obj_t self_in, mp_uint_t len); +void mp_obj_list_set_len(mp_obj_t self_in, size_t len); void mp_obj_list_store(mp_obj_t self_in, mp_obj_t index, mp_obj_t value); mp_obj_t mp_obj_list_sort(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); diff --git a/py/objlist.c b/py/objlist.c index 6d4a20a507..210140388a 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -33,8 +33,8 @@ #include "py/runtime.h" #include "py/stackctrl.h" -STATIC mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, mp_uint_t cur); -STATIC mp_obj_list_t *list_new(mp_uint_t n); +STATIC mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur); +STATIC mp_obj_list_t *list_new(size_t n); STATIC mp_obj_t list_extend(mp_obj_t self_in, mp_obj_t arg_in); STATIC mp_obj_t list_pop(size_t n_args, const mp_obj_t *args); @@ -50,7 +50,7 @@ STATIC void list_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t k kind = PRINT_REPR; } mp_print_str(print, "["); - for (mp_uint_t i = 0; i < o->len; i++) { + for (size_t i = 0; i < o->len; i++) { if (i > 0) { mp_print_str(print, ", "); } @@ -374,7 +374,7 @@ STATIC mp_obj_t list_insert(mp_obj_t self_in, mp_obj_t idx, mp_obj_t obj) { if (index < 0) { index = 0; } - if ((mp_uint_t)index > self->len) { + if ((size_t)index > self->len) { index = self->len; } @@ -451,7 +451,7 @@ const mp_obj_type_t mp_type_list = { .locals_dict = (mp_obj_dict_t*)&list_locals_dict, }; -void mp_obj_list_init(mp_obj_list_t *o, mp_uint_t n) { +void mp_obj_list_init(mp_obj_list_t *o, size_t n) { o->base.type = &mp_type_list; o->alloc = n < LIST_MIN_ALLOC ? LIST_MIN_ALLOC : n; o->len = n; @@ -459,16 +459,16 @@ void mp_obj_list_init(mp_obj_list_t *o, mp_uint_t n) { mp_seq_clear(o->items, n, o->alloc, sizeof(*o->items)); } -STATIC mp_obj_list_t *list_new(mp_uint_t n) { +STATIC mp_obj_list_t *list_new(size_t n) { mp_obj_list_t *o = m_new_obj(mp_obj_list_t); mp_obj_list_init(o, n); return o; } -mp_obj_t mp_obj_new_list(mp_uint_t n, mp_obj_t *items) { +mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items) { mp_obj_list_t *o = list_new(n); if (items != NULL) { - for (mp_uint_t i = 0; i < n; i++) { + for (size_t i = 0; i < n; i++) { o->items[i] = items[i]; } } @@ -481,7 +481,7 @@ void mp_obj_list_get(mp_obj_t self_in, mp_uint_t *len, mp_obj_t **items) { *items = self->items; } -void mp_obj_list_set_len(mp_obj_t self_in, mp_uint_t len) { +void mp_obj_list_set_len(mp_obj_t self_in, size_t len) { // trust that the caller knows what it's doing // TODO realloc if len got much smaller than alloc mp_obj_list_t *self = MP_OBJ_TO_PTR(self_in); @@ -501,7 +501,7 @@ typedef struct _mp_obj_list_it_t { mp_obj_base_t base; mp_fun_1_t iternext; mp_obj_t list; - mp_uint_t cur; + size_t cur; } mp_obj_list_it_t; STATIC mp_obj_t list_it_iternext(mp_obj_t self_in) { @@ -516,7 +516,7 @@ STATIC mp_obj_t list_it_iternext(mp_obj_t self_in) { } } -mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, mp_uint_t cur) { +mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur) { mp_obj_list_it_t *o = m_new_obj(mp_obj_list_it_t); o->base.type = &mp_type_polymorph_iter; o->iternext = list_it_iternext; diff --git a/py/objlist.h b/py/objlist.h index 443ede5743..5b2d216fc2 100644 --- a/py/objlist.h +++ b/py/objlist.h @@ -30,8 +30,8 @@ typedef struct _mp_obj_list_t { mp_obj_base_t base; - mp_uint_t alloc; - mp_uint_t len; + size_t alloc; + size_t len; mp_obj_t *items; } mp_obj_list_t; From 1ea2f7a8ce0353e423253135e417c02dc6466405 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:15:04 +1100 Subject: [PATCH 50/92] py/objdict: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 6 +++--- py/objdict.c | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/py/obj.h b/py/obj.h index fb35b67aa4..ab04107500 100644 --- a/py/obj.h +++ b/py/obj.h @@ -632,7 +632,7 @@ mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed, const mp_obj_t *closed); mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items); -mp_obj_t mp_obj_new_dict(mp_uint_t n_args); +mp_obj_t mp_obj_new_dict(size_t n_args); mp_obj_t mp_obj_new_set(mp_uint_t n_args, mp_obj_t *items); mp_obj_t mp_obj_new_slice(mp_obj_t start, mp_obj_t stop, mp_obj_t step); mp_obj_t mp_obj_new_super(mp_obj_t type, mp_obj_t obj); @@ -733,8 +733,8 @@ typedef struct _mp_obj_dict_t { mp_obj_base_t base; mp_map_t map; } mp_obj_dict_t; -void mp_obj_dict_init(mp_obj_dict_t *dict, mp_uint_t n_args); -mp_uint_t mp_obj_dict_len(mp_obj_t self_in); +void mp_obj_dict_init(mp_obj_dict_t *dict, size_t n_args); +size_t mp_obj_dict_len(mp_obj_t self_in); mp_obj_t mp_obj_dict_get(mp_obj_t self_in, mp_obj_t index); mp_obj_t mp_obj_dict_store(mp_obj_t self_in, mp_obj_t key, mp_obj_t value); mp_obj_t mp_obj_dict_delete(mp_obj_t self_in, mp_obj_t key); diff --git a/py/objdict.c b/py/objdict.c index 4942d37791..640df7b62a 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -41,11 +41,11 @@ STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwarg // This is a helper function to iterate through a dictionary. The state of // the iteration is held in *cur and should be initialised with zero for the // first call. Will return NULL when no more elements are available. -STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, mp_uint_t *cur) { - mp_uint_t max = dict->map.alloc; +STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) { + size_t max = dict->map.alloc; mp_map_t *map = &dict->map; - for (mp_uint_t i = *cur; i < max; i++) { + for (size_t i = *cur; i < max; i++) { if (MP_MAP_SLOT_IS_FILLED(map, i)) { *cur = i + 1; return &(map->table[i]); @@ -65,7 +65,7 @@ STATIC void dict_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_ mp_printf(print, "%q(", self->base.type->name); } mp_print_str(print, "{"); - mp_uint_t cur = 0; + size_t cur = 0; mp_map_elem_t *next = NULL; while ((next = dict_iter_next(self, &cur)) != NULL) { if (!first) { @@ -121,7 +121,7 @@ STATIC mp_obj_t dict_binary_op(mp_uint_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { if (MP_UNLIKELY(MP_OBJ_IS_TYPE(lhs_in, &mp_type_ordereddict) && MP_OBJ_IS_TYPE(rhs_in, &mp_type_ordereddict))) { // Iterate through both dictionaries simultaneously and compare keys and values. mp_obj_dict_t *rhs = MP_OBJ_TO_PTR(rhs_in); - mp_uint_t c1 = 0, c2 = 0; + size_t c1 = 0, c2 = 0; mp_map_elem_t *e1 = dict_iter_next(o, &c1), *e2 = dict_iter_next(rhs, &c2); for (; e1 != NULL && e2 != NULL; e1 = dict_iter_next(o, &c1), e2 = dict_iter_next(rhs, &c2)) { if (!mp_obj_equal(e1->key, e2->key) || !mp_obj_equal(e1->value, e2->value)) { @@ -137,7 +137,7 @@ STATIC mp_obj_t dict_binary_op(mp_uint_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { return mp_const_false; } - mp_uint_t cur = 0; + size_t cur = 0; mp_map_elem_t *next = NULL; while ((next = dict_iter_next(o, &cur)) != NULL) { mp_map_elem_t *elem = mp_map_lookup(&rhs->map, next->key, MP_MAP_LOOKUP); @@ -196,7 +196,7 @@ typedef struct _mp_obj_dict_it_t { mp_obj_base_t base; mp_fun_1_t iternext; mp_obj_t dict; - mp_uint_t cur; + size_t cur; } mp_obj_dict_it_t; STATIC mp_obj_t dict_it_iternext(mp_obj_t self_in) { @@ -340,7 +340,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(dict_setdefault_obj, 2, 3, dict_setde STATIC mp_obj_t dict_popitem(mp_obj_t self_in) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); - mp_uint_t cur = 0; + size_t cur = 0; mp_map_elem_t *next = dict_iter_next(self, &cur); if (next == NULL) { mp_raise_msg(&mp_type_KeyError, "popitem(): dictionary is empty"); @@ -367,7 +367,7 @@ STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwarg if (MP_OBJ_IS_DICT_TYPE(args[1])) { // update from other dictionary (make sure other is not self) if (args[1] != args[0]) { - mp_uint_t cur = 0; + size_t cur = 0; mp_map_elem_t *elem = NULL; while ((elem = dict_iter_next((mp_obj_dict_t*)MP_OBJ_TO_PTR(args[1]), &cur)) != NULL) { mp_map_lookup(&self->map, elem->key, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = elem->value; @@ -394,7 +394,7 @@ STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwarg } // update the dict with any keyword args - for (mp_uint_t i = 0; i < kwargs->alloc; i++) { + for (size_t i = 0; i < kwargs->alloc; i++) { if (MP_MAP_SLOT_IS_FILLED(kwargs, i)) { mp_map_lookup(&self->map, kwargs->table[i].key, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = kwargs->table[i].value; } @@ -423,7 +423,7 @@ typedef struct _mp_obj_dict_view_it_t { mp_obj_base_t base; mp_dict_view_kind_t kind; mp_obj_t dict; - mp_uint_t cur; + size_t cur; } mp_obj_dict_view_it_t; typedef struct _mp_obj_dict_view_t { @@ -590,18 +590,18 @@ const mp_obj_type_t mp_type_ordereddict = { }; #endif -void mp_obj_dict_init(mp_obj_dict_t *dict, mp_uint_t n_args) { +void mp_obj_dict_init(mp_obj_dict_t *dict, size_t n_args) { dict->base.type = &mp_type_dict; mp_map_init(&dict->map, n_args); } -mp_obj_t mp_obj_new_dict(mp_uint_t n_args) { +mp_obj_t mp_obj_new_dict(size_t n_args) { mp_obj_dict_t *o = m_new_obj(mp_obj_dict_t); mp_obj_dict_init(o, n_args); return MP_OBJ_FROM_PTR(o); } -mp_uint_t mp_obj_dict_len(mp_obj_t self_in) { +size_t mp_obj_dict_len(mp_obj_t self_in) { mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); return self->map.used; } From 68cd3a93f0728b61b0d2407f4ae802806dddce4d Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:16:33 +1100 Subject: [PATCH 51/92] py/objset: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 2 +- py/objset.c | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/py/obj.h b/py/obj.h index ab04107500..2d02624421 100644 --- a/py/obj.h +++ b/py/obj.h @@ -633,7 +633,7 @@ mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed, const mp_obj_t *cl mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items); mp_obj_t mp_obj_new_dict(size_t n_args); -mp_obj_t mp_obj_new_set(mp_uint_t n_args, mp_obj_t *items); +mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items); mp_obj_t mp_obj_new_slice(mp_obj_t start, mp_obj_t stop, mp_obj_t step); mp_obj_t mp_obj_new_super(mp_obj_t type, mp_obj_t obj); mp_obj_t mp_obj_new_bound_meth(mp_obj_t meth, mp_obj_t self); diff --git a/py/objset.c b/py/objset.c index 778bbe15b8..b9da44a44f 100644 --- a/py/objset.c +++ b/py/objset.c @@ -44,7 +44,7 @@ typedef struct _mp_obj_set_it_t { mp_obj_base_t base; mp_fun_1_t iternext; mp_obj_set_t *set; - mp_uint_t cur; + size_t cur; } mp_obj_set_it_t; STATIC mp_obj_t set_it_iternext(mp_obj_t self_in); @@ -96,7 +96,7 @@ STATIC void set_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t } #endif mp_print_str(print, "{"); - for (mp_uint_t i = 0; i < self->set.alloc; i++) { + for (size_t i = 0; i < self->set.alloc; i++) { if (MP_SET_SLOT_IS_FILLED(&self->set, i)) { if (!first) { mp_print_str(print, ", "); @@ -143,10 +143,10 @@ STATIC mp_obj_t set_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ STATIC mp_obj_t set_it_iternext(mp_obj_t self_in) { mp_obj_set_it_t *self = MP_OBJ_TO_PTR(self_in); - mp_uint_t max = self->set->set.alloc; + size_t max = self->set->set.alloc; mp_set_t *set = &self->set->set; - for (mp_uint_t i = self->cur; i < max; i++) { + for (size_t i = self->cur; i < max; i++) { if (MP_SET_SLOT_IS_FILLED(set, i)) { self->cur = i + 1; return set->table[i]; @@ -228,7 +228,7 @@ STATIC mp_obj_t set_diff_int(size_t n_args, const mp_obj_t *args, bool update) { } - for (mp_uint_t i = 1; i < n_args; i++) { + for (size_t i = 1; i < n_args; i++) { mp_obj_t other = args[i]; if (self == other) { set_clear(self); @@ -436,7 +436,7 @@ STATIC void set_update_int(mp_obj_set_t *self, mp_obj_t other_in) { STATIC mp_obj_t set_update(size_t n_args, const mp_obj_t *args) { check_set(args[0]); - for (mp_uint_t i = 1; i < n_args; i++) { + for (size_t i = 1; i < n_args; i++) { set_update_int(MP_OBJ_TO_PTR(args[0]), args[i]); } @@ -462,10 +462,10 @@ STATIC mp_obj_t set_unary_op(mp_uint_t op, mp_obj_t self_in) { if (MP_OBJ_IS_TYPE(self_in, &mp_type_frozenset)) { // start hash with unique value mp_int_t hash = (mp_int_t)(uintptr_t)&mp_type_frozenset; - mp_uint_t max = self->set.alloc; + size_t max = self->set.alloc; mp_set_t *set = &self->set; - for (mp_uint_t i = 0; i < max; i++) { + for (size_t i = 0; i < max; i++) { if (MP_SET_SLOT_IS_FILLED(set, i)) { hash += MP_OBJ_SMALL_INT_VALUE(mp_unary_op(MP_UNARY_OP_HASH, set->table[i])); } @@ -587,11 +587,11 @@ const mp_obj_type_t mp_type_frozenset = { }; #endif -mp_obj_t mp_obj_new_set(mp_uint_t n_args, mp_obj_t *items) { +mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items) { mp_obj_set_t *o = m_new_obj(mp_obj_set_t); o->base.type = &mp_type_set; mp_set_init(&o->set, n_args); - for (mp_uint_t i = 0; i < n_args; i++) { + for (size_t i = 0; i < n_args; i++) { mp_set_lookup(&o->set, items[i], MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); } return MP_OBJ_FROM_PTR(o); From c0d9500eeefe57076f2f9a36afaa80186e9e30be Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:26:48 +1100 Subject: [PATCH 52/92] py/objstr: Convert mp_uint_t to size_t (and use int) where appropriate. --- py/obj.h | 6 +++--- py/objstr.c | 54 +++++++++++++++++++++++----------------------- py/objstr.h | 4 ++-- py/objstrunicode.c | 2 +- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/py/obj.h b/py/obj.h index 2d02624421..efb9234132 100644 --- a/py/obj.h +++ b/py/obj.h @@ -610,9 +610,9 @@ mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value); mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base); mp_obj_t mp_obj_new_int_from_ll(long long val); // this must return a multi-precision integer object (or raise an overflow exception) mp_obj_t mp_obj_new_int_from_ull(unsigned long long val); // this must return a multi-precision integer object (or raise an overflow exception) -mp_obj_t mp_obj_new_str(const char* data, mp_uint_t len, bool make_qstr_if_not_already); +mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already); mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr); -mp_obj_t mp_obj_new_bytes(const byte* data, mp_uint_t len); +mp_obj_t mp_obj_new_bytes(const byte* data, size_t len); mp_obj_t mp_obj_new_bytearray(mp_uint_t n, void *items); mp_obj_t mp_obj_new_bytearray_by_ref(mp_uint_t n, void *items); #if MICROPY_PY_BUILTINS_FLOAT @@ -700,7 +700,7 @@ qstr mp_obj_str_get_qstr(mp_obj_t self_in); // use this if you will anyway conve const char *mp_obj_str_get_str(mp_obj_t self_in); // use this only if you need the string to be null terminated const char *mp_obj_str_get_data(mp_obj_t self_in, mp_uint_t *len); mp_obj_t mp_obj_str_intern(mp_obj_t str); -void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, mp_uint_t str_len, bool is_bytes); +void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, size_t str_len, bool is_bytes); #if MICROPY_PY_BUILTINS_FLOAT // float diff --git a/py/objstr.c b/py/objstr.c index b437d97950..262b89ddd8 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -44,7 +44,7 @@ STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); /******************************************************************************/ /* str */ -void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, mp_uint_t str_len, bool is_bytes) { +void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, size_t str_len, bool is_bytes) { // this escapes characters, but it will be very slow to print (calling print many times) bool has_single_quote = false; bool has_double_quote = false; @@ -251,9 +251,9 @@ wrong_args: // like strstr but with specified length and allows \0 bytes // TODO replace with something more efficient/standard -const byte *find_subbytes(const byte *haystack, mp_uint_t hlen, const byte *needle, mp_uint_t nlen, mp_int_t direction) { +const byte *find_subbytes(const byte *haystack, size_t hlen, const byte *needle, size_t nlen, int direction) { if (hlen >= nlen) { - mp_uint_t str_index, str_index_end; + size_t str_index, str_index_end; if (direction > 0) { str_index = 0; str_index_end = hlen - nlen; @@ -333,7 +333,7 @@ mp_obj_t mp_obj_str_binary_op(mp_uint_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { // size and execution time so we don't. const byte *rhs_data; - mp_uint_t rhs_len; + size_t rhs_len; if (lhs_type == mp_obj_get_type(rhs_in)) { GET_STR_DATA_LEN(rhs_in, rhs_data_, rhs_len_); rhs_data = rhs_data_; @@ -441,8 +441,8 @@ STATIC mp_obj_t str_join(mp_obj_t self_in, mp_obj_t arg) { } // count required length - mp_uint_t required_len = 0; - for (mp_uint_t i = 0; i < seq_len; i++) { + size_t required_len = 0; + for (size_t i = 0; i < seq_len; i++) { if (mp_obj_get_type(seq_items[i]) != self_type) { mp_raise_TypeError( "join expects a list of str/bytes objects consistent with self object"); @@ -458,7 +458,7 @@ STATIC mp_obj_t str_join(mp_obj_t self_in, mp_obj_t arg) { vstr_t vstr; vstr_init_len(&vstr, required_len); byte *data = (byte*)vstr.buf; - for (mp_uint_t i = 0; i < seq_len; i++) { + for (size_t i = 0; i < seq_len; i++) { if (i > 0) { memcpy(data, sep_str, sep_len); data += sep_len; @@ -644,7 +644,7 @@ STATIC mp_obj_t str_rsplit(size_t n_args, const mp_obj_t *args) { } if (idx != 0) { // We split less parts than split limit, now go cleanup surplus - mp_int_t used = org_splits + 1 - idx; + size_t used = org_splits + 1 - idx; memmove(res->items, &res->items[idx], used * sizeof(mp_obj_t)); mp_seq_clear(res->items, used, res->alloc, sizeof(*res->items)); res->len = used; @@ -654,7 +654,7 @@ STATIC mp_obj_t str_rsplit(size_t n_args, const mp_obj_t *args) { return MP_OBJ_FROM_PTR(res); } -STATIC mp_obj_t str_finder(size_t n_args, const mp_obj_t *args, mp_int_t direction, bool is_index) { +STATIC mp_obj_t str_finder(size_t n_args, const mp_obj_t *args, int direction, bool is_index) { const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); mp_check_self(MP_OBJ_IS_STR_OR_BYTES(args[0])); @@ -762,16 +762,16 @@ STATIC mp_obj_t str_uni_strip(int type, size_t n_args, const mp_obj_t *args) { GET_STR_DATA_LEN(args[0], orig_str, orig_str_len); - mp_uint_t first_good_char_pos = 0; + size_t first_good_char_pos = 0; bool first_good_char_pos_set = false; - mp_uint_t last_good_char_pos = 0; - mp_uint_t i = 0; - mp_int_t delta = 1; + size_t last_good_char_pos = 0; + size_t i = 0; + int delta = 1; if (type == RSTRIP) { i = orig_str_len - 1; delta = -1; } - for (mp_uint_t len = orig_str_len; len > 0; len--) { + for (size_t len = orig_str_len; len > 0; len--) { if (find_subbytes(chars_to_del, chars_to_del_len, &orig_str[i], 1, 1) == NULL) { if (!first_good_char_pos_set) { first_good_char_pos_set = true; @@ -801,7 +801,7 @@ STATIC mp_obj_t str_uni_strip(int type, size_t n_args, const mp_obj_t *args) { assert(last_good_char_pos >= first_good_char_pos); //+1 to accomodate the last character - mp_uint_t stripped_len = last_good_char_pos - first_good_char_pos + 1; + size_t stripped_len = last_good_char_pos - first_good_char_pos + 1; if (stripped_len == orig_str_len) { // If nothing was stripped, don't bother to dup original string // TODO: watch out for this case when we'll get to bytearray.strip() @@ -1588,11 +1588,11 @@ STATIC mp_obj_t str_replace(size_t n_args, const mp_obj_t *args) { // first pass computes the required length of the replaced string // second pass does the replacements for (;;) { - mp_uint_t replaced_str_index = 0; - mp_uint_t num_replacements_done = 0; + size_t replaced_str_index = 0; + size_t num_replacements_done = 0; const byte *old_occurrence; const byte *offset_ptr = str; - mp_uint_t str_len_remain = str_len; + size_t str_len_remain = str_len; if (old_len == 0) { // if old_str is empty, copy new_str to start of replaced string // copy the replacement string @@ -1602,7 +1602,7 @@ STATIC mp_obj_t str_replace(size_t n_args, const mp_obj_t *args) { replaced_str_index += new_len; num_replacements_done++; } - while (num_replacements_done != (mp_uint_t)max_rep && str_len_remain > 0 && (old_occurrence = find_subbytes(offset_ptr, str_len_remain, old, old_len, 1)) != NULL) { + while (num_replacements_done != (size_t)max_rep && str_len_remain > 0 && (old_occurrence = find_subbytes(offset_ptr, str_len_remain, old, old_len, 1)) != NULL) { if (old_len == 0) { old_occurrence += 1; } @@ -1688,7 +1688,7 @@ STATIC mp_obj_t str_count(size_t n_args, const mp_obj_t *args) { } #if MICROPY_PY_BUILTINS_STR_PARTITION -STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, mp_int_t direction) { +STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, int direction) { mp_check_self(MP_OBJ_IS_STR_OR_BYTES(self_in)); mp_obj_type_t *self_type = mp_obj_get_type(self_in); if (self_type != mp_obj_get_type(arg)) { @@ -1721,7 +1721,7 @@ STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, mp_int_t directi const byte *position_ptr = find_subbytes(str, str_len, sep, sep_len, direction); if (position_ptr != NULL) { - mp_uint_t position = position_ptr - str; + size_t position = position_ptr - str; result[0] = mp_obj_new_str_of_type(self_type, str, position); result[1] = arg; result[2] = mp_obj_new_str_of_type(self_type, str + position + sep_len, str_len - position - sep_len); @@ -1745,7 +1745,7 @@ STATIC mp_obj_t str_caseconv(unichar (*op)(unichar), mp_obj_t self_in) { vstr_t vstr; vstr_init_len(&vstr, self_len); byte *data = (byte*)vstr.buf; - for (mp_uint_t i = 0; i < self_len; i++) { + for (size_t i = 0; i < self_len; i++) { *data++ = op(*self_data++); } return mp_obj_new_str_from_vstr(mp_obj_get_type(self_in), &vstr); @@ -1767,7 +1767,7 @@ STATIC mp_obj_t str_uni_istype(bool (*f)(unichar), mp_obj_t self_in) { } if (f != unichar_isupper && f != unichar_islower) { - for (mp_uint_t i = 0; i < self_len; i++) { + for (size_t i = 0; i < self_len; i++) { if (!f(*self_data++)) { return mp_const_false; } @@ -1775,7 +1775,7 @@ STATIC mp_obj_t str_uni_istype(bool (*f)(unichar), mp_obj_t self_in) { } else { bool contains_alpha = false; - for (mp_uint_t i = 0; i < self_len; i++) { // only check alphanumeric characters + for (size_t i = 0; i < self_len; i++) { // only check alphanumeric characters if (unichar_isalpha(*self_data++)) { contains_alpha = true; if (!f(*(self_data - 1))) { // -1 because we already incremented above @@ -2019,7 +2019,7 @@ mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr) { return MP_OBJ_FROM_PTR(o); } -mp_obj_t mp_obj_new_str(const char* data, mp_uint_t len, bool make_qstr_if_not_already) { +mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already) { if (make_qstr_if_not_already) { // use existing, or make a new qstr return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); @@ -2040,7 +2040,7 @@ mp_obj_t mp_obj_str_intern(mp_obj_t str) { return MP_OBJ_NEW_QSTR(qstr_from_strn((const char*)data, len)); } -mp_obj_t mp_obj_new_bytes(const byte* data, mp_uint_t len) { +mp_obj_t mp_obj_new_bytes(const byte* data, size_t len) { return mp_obj_new_str_of_type(&mp_type_bytes, data, len); } @@ -2126,7 +2126,7 @@ typedef struct _mp_obj_str8_it_t { mp_obj_base_t base; mp_fun_1_t iternext; mp_obj_t str; - mp_uint_t cur; + size_t cur; } mp_obj_str8_it_t; #if !MICROPY_PY_BUILTINS_STR_UNICODE diff --git a/py/objstr.h b/py/objstr.h index ad2777afbc..e92832d106 100644 --- a/py/objstr.h +++ b/py/objstr.h @@ -32,7 +32,7 @@ typedef struct _mp_obj_str_t { mp_obj_base_t base; mp_uint_t hash; // len == number of bytes used in data, alloc = len + 1 because (at the moment) we also append a null byte - mp_uint_t len; + size_t len; const byte *data; } mp_obj_str_t; @@ -72,7 +72,7 @@ mp_int_t mp_obj_str_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_u const byte *str_index_to_ptr(const mp_obj_type_t *type, const byte *self_data, size_t self_len, mp_obj_t index, bool is_slice); -const byte *find_subbytes(const byte *haystack, mp_uint_t hlen, const byte *needle, mp_uint_t nlen, mp_int_t direction); +const byte *find_subbytes(const byte *haystack, size_t hlen, const byte *needle, size_t nlen, int direction); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(str_encode_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(str_find_obj); diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 8444a26892..0091edd725 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -284,7 +284,7 @@ typedef struct _mp_obj_str_it_t { mp_obj_base_t base; mp_fun_1_t iternext; mp_obj_t str; - mp_uint_t cur; + size_t cur; } mp_obj_str_it_t; STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { From ccc5254224801392e580cfc00b6cdd098fb7ad4a Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:31:43 +1100 Subject: [PATCH 53/92] py/objarray: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 6 +++--- py/objarray.c | 32 ++++++++++++++++---------------- py/objarray.h | 6 +++--- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/py/obj.h b/py/obj.h index efb9234132..a507a1096c 100644 --- a/py/obj.h +++ b/py/obj.h @@ -613,8 +613,8 @@ mp_obj_t mp_obj_new_int_from_ull(unsigned long long val); // this must return a mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already); mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr); mp_obj_t mp_obj_new_bytes(const byte* data, size_t len); -mp_obj_t mp_obj_new_bytearray(mp_uint_t n, void *items); -mp_obj_t mp_obj_new_bytearray_by_ref(mp_uint_t n, void *items); +mp_obj_t mp_obj_new_bytearray(size_t n, void *items); +mp_obj_t mp_obj_new_bytearray_by_ref(size_t n, void *items); #if MICROPY_PY_BUILTINS_FLOAT mp_obj_t mp_obj_new_int_from_float(mp_float_t val); mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag); @@ -639,7 +639,7 @@ mp_obj_t mp_obj_new_super(mp_obj_t type, mp_obj_t obj); mp_obj_t mp_obj_new_bound_meth(mp_obj_t meth, mp_obj_t self); mp_obj_t mp_obj_new_getitem_iter(mp_obj_t *args); mp_obj_t mp_obj_new_module(qstr module_name); -mp_obj_t mp_obj_new_memoryview(byte typecode, mp_uint_t nitems, void *items); +mp_obj_t mp_obj_new_memoryview(byte typecode, size_t nitems, void *items); mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in); const char *mp_obj_get_type_str(mp_const_obj_t o_in); diff --git a/py/objarray.c b/py/objarray.c index ed666df8f0..f237918574 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -56,7 +56,7 @@ #if MICROPY_PY_BUILTINS_MEMORYVIEW #define TYPECODE_MASK (0x7f) #else -#define TYPECODE_MASK (~(mp_uint_t)0) +#define TYPECODE_MASK (~(size_t)0) #endif STATIC mp_obj_t array_iterator_new(mp_obj_t array_in); @@ -78,7 +78,7 @@ STATIC void array_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_printf(print, "array('%c'", o->typecode); if (o->len > 0) { mp_print_str(print, ", ["); - for (mp_uint_t i = 0; i < o->len; i++) { + for (size_t i = 0; i < o->len; i++) { if (i > 0) { mp_print_str(print, ", "); } @@ -92,7 +92,7 @@ STATIC void array_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t #endif #if MICROPY_PY_BUILTINS_BYTEARRAY || MICROPY_PY_ARRAY -STATIC mp_obj_array_t *array_new(char typecode, mp_uint_t n) { +STATIC mp_obj_array_t *array_new(char typecode, size_t n) { int typecode_size = mp_binary_get_size('@', typecode, NULL); mp_obj_array_t *o = m_new_obj(mp_obj_array_t); #if MICROPY_PY_BUILTINS_BYTEARRAY && MICROPY_PY_ARRAY @@ -124,13 +124,13 @@ STATIC mp_obj_t array_construct(char typecode, mp_obj_t initializer) { // construct array from raw bytes // we round-down the len to make it a multiple of sz (CPython raises error) size_t sz = mp_binary_get_size('@', typecode, NULL); - mp_uint_t len = bufinfo.len / sz; + size_t len = bufinfo.len / sz; mp_obj_array_t *o = array_new(typecode, len); memcpy(o->items, bufinfo.buf, len * sz); return MP_OBJ_FROM_PTR(o); } - mp_uint_t len; + size_t len; // Try to create array of exact len if initializer len is known mp_obj_t len_in = mp_obj_len_maybe(initializer); if (len_in == MP_OBJ_NULL) { @@ -143,7 +143,7 @@ STATIC mp_obj_t array_construct(char typecode, mp_obj_t initializer) { mp_obj_t iterable = mp_getiter(initializer); mp_obj_t item; - mp_uint_t i = 0; + size_t i = 0; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (len == 0) { array_append(MP_OBJ_FROM_PTR(array), item); @@ -198,7 +198,7 @@ STATIC mp_obj_t bytearray_make_new(const mp_obj_type_t *type_in, size_t n_args, #if MICROPY_PY_BUILTINS_MEMORYVIEW -mp_obj_t mp_obj_new_memoryview(byte typecode, mp_uint_t nitems, void *items) { +mp_obj_t mp_obj_new_memoryview(byte typecode, size_t nitems, void *items) { mp_obj_array_t *self = m_new_obj(mp_obj_array_t); self->base.type = &mp_type_memoryview; self->typecode = typecode; @@ -254,7 +254,7 @@ STATIC mp_obj_t array_binary_op(mp_uint_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) size_t sz = mp_binary_get_size('@', lhs_bufinfo.typecode, NULL); // convert byte count to element count (in case rhs is not multiple of sz) - mp_uint_t rhs_len = rhs_bufinfo.len / sz; + size_t rhs_len = rhs_bufinfo.len / sz; // note: lhs->len is element count of lhs, lhs_bufinfo.len is byte count mp_obj_array_t *res = array_new(lhs_bufinfo.typecode, lhs->len + rhs_len); @@ -345,7 +345,7 @@ STATIC mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in) { size_t sz = mp_binary_get_size('@', self->typecode, NULL); // convert byte count to element count - mp_uint_t len = arg_bufinfo.len / sz; + size_t len = arg_bufinfo.len / sz; // make sure we have enough room to extend // TODO: alloc policy; at the moment we go conservative @@ -384,7 +384,7 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value if (value != MP_OBJ_SENTINEL) { #if MICROPY_PY_ARRAY_SLICE_ASSIGN // Assign - mp_uint_t src_len; + size_t src_len; void *src_items; size_t item_sz = mp_binary_get_size('@', o->typecode & TYPECODE_MASK, NULL); if (MP_OBJ_IS_OBJ(value) && ((mp_obj_base_t*)MP_OBJ_TO_PTR(value))->type->subscr == array_subscr) { @@ -501,7 +501,7 @@ STATIC mp_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, mp_ui // read-only memoryview return 1; } - bufinfo->buf = (uint8_t*)bufinfo->buf + (mp_uint_t)o->free * sz; + bufinfo->buf = (uint8_t*)bufinfo->buf + (size_t)o->free * sz; } #else (void)flags; @@ -562,20 +562,20 @@ const mp_obj_type_t mp_type_memoryview = { #endif /* unused -mp_uint_t mp_obj_array_len(mp_obj_t self_in) { +size_t mp_obj_array_len(mp_obj_t self_in) { return ((mp_obj_array_t *)self_in)->len; } */ #if MICROPY_PY_BUILTINS_BYTEARRAY -mp_obj_t mp_obj_new_bytearray(mp_uint_t n, void *items) { +mp_obj_t mp_obj_new_bytearray(size_t n, void *items) { mp_obj_array_t *o = array_new(BYTEARRAY_TYPECODE, n); memcpy(o->items, items, n); return MP_OBJ_FROM_PTR(o); } // Create bytearray which references specified memory area -mp_obj_t mp_obj_new_bytearray_by_ref(mp_uint_t n, void *items) { +mp_obj_t mp_obj_new_bytearray_by_ref(size_t n, void *items) { mp_obj_array_t *o = m_new_obj(mp_obj_array_t); o->base.type = &mp_type_bytearray; o->typecode = BYTEARRAY_TYPECODE; @@ -592,8 +592,8 @@ mp_obj_t mp_obj_new_bytearray_by_ref(mp_uint_t n, void *items) { typedef struct _mp_obj_array_it_t { mp_obj_base_t base; mp_obj_array_t *array; - mp_uint_t offset; - mp_uint_t cur; + size_t offset; + size_t cur; } mp_obj_array_it_t; STATIC mp_obj_t array_it_iternext(mp_obj_t self_in) { diff --git a/py/objarray.h b/py/objarray.h index 013ac5be9b..06a2a07efb 100644 --- a/py/objarray.h +++ b/py/objarray.h @@ -32,11 +32,11 @@ typedef struct _mp_obj_array_t { mp_obj_base_t base; - mp_uint_t typecode : 8; + size_t typecode : 8; // free is number of unused elements after len used elements // alloc size = len + free - mp_uint_t free : (8 * sizeof(mp_uint_t) - 8); - mp_uint_t len; // in elements + size_t free : (8 * sizeof(size_t) - 8); + size_t len; // in elements void *items; } mp_obj_array_t; From dbcdb9f8d8d5dd56ede1f0705355679a11b75658 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:34:53 +1100 Subject: [PATCH 54/92] py/objfun: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 4 ++-- py/objfun.c | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/py/obj.h b/py/obj.h index a507a1096c..80294b3b4f 100644 --- a/py/obj.h +++ b/py/obj.h @@ -626,8 +626,8 @@ mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table); mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table); -mp_obj_t mp_obj_new_fun_viper(mp_uint_t n_args, void *fun_data, mp_uint_t type_sig); -mp_obj_t mp_obj_new_fun_asm(mp_uint_t n_args, void *fun_data, mp_uint_t type_sig); +mp_obj_t mp_obj_new_fun_viper(size_t n_args, void *fun_data, mp_uint_t type_sig); +mp_obj_t mp_obj_new_fun_asm(size_t n_args, void *fun_data, mp_uint_t type_sig); mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed, const mp_obj_t *closed); mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); diff --git a/py/objfun.c b/py/objfun.c index 207e68a771..e5f6009dc1 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -181,9 +181,9 @@ STATIC void fun_bc_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t #endif #if DEBUG_PRINT -STATIC void dump_args(const mp_obj_t *a, mp_uint_t sz) { +STATIC void dump_args(const mp_obj_t *a, size_t sz) { DEBUG_printf("%p: ", a); - for (mp_uint_t i = 0; i < sz; i++) { + for (size_t i = 0; i < sz; i++) { DEBUG_printf("%p ", a[i]); } DEBUG_printf("\n"); @@ -247,15 +247,15 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const const byte *ip = self->bytecode; // bytecode prelude: state size and exception stack size - mp_uint_t n_state = mp_decode_uint(&ip); - mp_uint_t n_exc_stack = mp_decode_uint(&ip); + size_t n_state = mp_decode_uint(&ip); + size_t n_exc_stack = mp_decode_uint(&ip); #if VM_DETECT_STACK_OVERFLOW n_state += 1; #endif // allocate state for locals and stack - mp_uint_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); + size_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); mp_code_state_t *code_state = NULL; if (state_size > VM_MAX_STATE_ON_STACK) { code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); @@ -288,7 +288,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const if (!(vm_return_kind == MP_VM_RETURN_EXCEPTION && self->n_pos_args + self->n_kwonly_args == 0)) { // Just check to see that we have at least 1 null object left in the state. bool overflow = true; - for (mp_uint_t i = 0; i < n_state - self->n_pos_args - self->n_kwonly_args; i++) { + for (size_t i = 0; i < n_state - self->n_pos_args - self->n_kwonly_args; i++) { if (code_state->state[i] == MP_OBJ_NULL) { overflow = false; break; @@ -350,8 +350,8 @@ const mp_obj_type_t mp_type_fun_bc = { }; mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args_in, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table) { - mp_uint_t n_def_args = 0; - mp_uint_t n_extra_args = 0; + size_t n_def_args = 0; + size_t n_extra_args = 0; mp_obj_tuple_t *def_args = MP_OBJ_TO_PTR(def_args_in); if (def_args_in != MP_OBJ_NULL) { assert(MP_OBJ_IS_TYPE(def_args_in, &mp_type_tuple)); @@ -409,7 +409,7 @@ mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const typedef struct _mp_obj_fun_viper_t { mp_obj_base_t base; - mp_uint_t n_args; + size_t n_args; void *fun_data; // GC must be able to trace this pointer mp_uint_t type_sig; } mp_obj_fun_viper_t; @@ -457,7 +457,7 @@ STATIC const mp_obj_type_t mp_type_fun_viper = { .unary_op = mp_generic_unary_op, }; -mp_obj_t mp_obj_new_fun_viper(mp_uint_t n_args, void *fun_data, mp_uint_t type_sig) { +mp_obj_t mp_obj_new_fun_viper(size_t n_args, void *fun_data, mp_uint_t type_sig) { mp_obj_fun_viper_t *o = m_new_obj(mp_obj_fun_viper_t); o->base.type = &mp_type_fun_viper; o->n_args = n_args; @@ -475,7 +475,7 @@ mp_obj_t mp_obj_new_fun_viper(mp_uint_t n_args, void *fun_data, mp_uint_t type_s typedef struct _mp_obj_fun_asm_t { mp_obj_base_t base; - mp_uint_t n_args; + size_t n_args; void *fun_data; // GC must be able to trace this pointer mp_uint_t type_sig; } mp_obj_fun_asm_t; @@ -573,7 +573,7 @@ STATIC const mp_obj_type_t mp_type_fun_asm = { .unary_op = mp_generic_unary_op, }; -mp_obj_t mp_obj_new_fun_asm(mp_uint_t n_args, void *fun_data, mp_uint_t type_sig) { +mp_obj_t mp_obj_new_fun_asm(size_t n_args, void *fun_data, mp_uint_t type_sig) { mp_obj_fun_asm_t *o = m_new_obj(mp_obj_fun_asm_t); o->base.type = &mp_type_fun_asm; o->n_args = n_args; From efa629028ad453206c04104735e33a79ae60c8a2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:36:04 +1100 Subject: [PATCH 55/92] py/objclosure: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 2 +- py/objclosure.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/py/obj.h b/py/obj.h index 80294b3b4f..d28623e64c 100644 --- a/py/obj.h +++ b/py/obj.h @@ -629,7 +629,7 @@ mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const mp_obj_t mp_obj_new_fun_viper(size_t n_args, void *fun_data, mp_uint_t type_sig); mp_obj_t mp_obj_new_fun_asm(size_t n_args, void *fun_data, mp_uint_t type_sig); mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); -mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed, const mp_obj_t *closed); +mp_obj_t mp_obj_new_closure(mp_obj_t fun, size_t n_closed, const mp_obj_t *closed); mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); mp_obj_t mp_obj_new_list(size_t n, mp_obj_t *items); mp_obj_t mp_obj_new_dict(size_t n_args); diff --git a/py/objclosure.c b/py/objclosure.c index 4b37d2dd12..3e12358bbd 100644 --- a/py/objclosure.c +++ b/py/objclosure.c @@ -32,7 +32,7 @@ typedef struct _mp_obj_closure_t { mp_obj_base_t base; mp_obj_t fun; - mp_uint_t n_closed; + size_t n_closed; mp_obj_t closed[]; } mp_obj_closure_t; @@ -41,7 +41,7 @@ STATIC mp_obj_t closure_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const // need to concatenate closed-over-vars and args - mp_uint_t n_total = self->n_closed + n_args + 2 * n_kw; + size_t n_total = self->n_closed + n_args + 2 * n_kw; if (n_total <= 5) { // use stack to allocate temporary args array mp_obj_t args2[5]; @@ -66,7 +66,7 @@ STATIC void closure_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_ mp_print_str(print, "fun, PRINT_REPR); mp_printf(print, " at %p, n_closed=%u ", o, (int)o->n_closed); - for (mp_uint_t i = 0; i < o->n_closed; i++) { + for (size_t i = 0; i < o->n_closed; i++) { if (o->closed[i] == MP_OBJ_NULL) { mp_print_str(print, "(nil)"); } else { @@ -87,7 +87,7 @@ const mp_obj_type_t closure_type = { .call = closure_call, }; -mp_obj_t mp_obj_new_closure(mp_obj_t fun, mp_uint_t n_closed_over, const mp_obj_t *closed) { +mp_obj_t mp_obj_new_closure(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed) { mp_obj_closure_t *o = m_new_obj_var(mp_obj_closure_t, mp_obj_t, n_closed_over); o->base.type = &closure_type; o->fun = fun; From fa5a591757569136a444715cce2248b63ad73512 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:38:14 +1100 Subject: [PATCH 56/92] py/objexcept: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 2 +- py/objexcept.c | 2 +- py/objexcept.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/py/obj.h b/py/obj.h index d28623e64c..1b926ce144 100644 --- a/py/obj.h +++ b/py/obj.h @@ -621,7 +621,7 @@ mp_obj_t mp_obj_new_complex(mp_float_t real, mp_float_t imag); #endif mp_obj_t mp_obj_new_exception(const mp_obj_type_t *exc_type); mp_obj_t mp_obj_new_exception_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg); -mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, mp_uint_t n_args, const mp_obj_t *args); +mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args); mp_obj_t mp_obj_new_exception_msg(const mp_obj_type_t *exc_type, const char *msg); mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, const char *fmt, ...); // counts args by number of % symbols in fmt, excluding %%; can only handle void* sizes (ie no float/double!) mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table); diff --git a/py/objexcept.c b/py/objexcept.c index 76d34f56e4..1f1009ff78 100644 --- a/py/objexcept.c +++ b/py/objexcept.c @@ -312,7 +312,7 @@ mp_obj_t mp_obj_new_exception_arg1(const mp_obj_type_t *exc_type, mp_obj_t arg) return mp_obj_new_exception_args(exc_type, 1, &arg); } -mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, mp_uint_t n_args, const mp_obj_t *args) { +mp_obj_t mp_obj_new_exception_args(const mp_obj_type_t *exc_type, size_t n_args, const mp_obj_t *args) { assert(exc_type->make_new == mp_obj_exception_make_new); return exc_type->make_new(exc_type, n_args, 0, args); } diff --git a/py/objexcept.h b/py/objexcept.h index 88bce2b370..3128fded79 100644 --- a/py/objexcept.h +++ b/py/objexcept.h @@ -31,8 +31,8 @@ typedef struct _mp_obj_exception_t { mp_obj_base_t base; - mp_uint_t traceback_alloc : (BITS_PER_WORD / 2); - mp_uint_t traceback_len : (BITS_PER_WORD / 2); + size_t traceback_alloc : (8 * sizeof(size_t) / 2); + size_t traceback_len : (8 * sizeof(size_t) / 2); size_t *traceback_data; mp_obj_tuple_t *args; } mp_obj_exception_t; From da36f5232dcf02c448ee9933e358b711cfe03f30 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:41:43 +1100 Subject: [PATCH 57/92] py/objint: Convert mp_uint_t to size_t where appropriate. --- py/obj.h | 2 +- py/objint.c | 2 +- py/objint_longlong.c | 2 +- py/objint_mpz.c | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/py/obj.h b/py/obj.h index 1b926ce144..1169a9095e 100644 --- a/py/obj.h +++ b/py/obj.h @@ -607,7 +607,7 @@ static inline mp_obj_t mp_obj_new_bool(mp_int_t x) { return x ? mp_const_true : mp_obj_t mp_obj_new_cell(mp_obj_t obj); mp_obj_t mp_obj_new_int(mp_int_t value); mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value); -mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base); +mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base); mp_obj_t mp_obj_new_int_from_ll(long long val); // this must return a multi-precision integer object (or raise an overflow exception) mp_obj_t mp_obj_new_int_from_ull(unsigned long long val); // this must return a multi-precision integer object (or raise an overflow exception) mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already); diff --git a/py/objint.c b/py/objint.c index 222aa1de71..2a7e3f3fd9 100644 --- a/py/objint.c +++ b/py/objint.c @@ -291,7 +291,7 @@ mp_obj_t mp_obj_int_binary_op(mp_uint_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { } // This is called only with strings whose value doesn't fit in SMALL_INT -mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base) { +mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { mp_raise_msg(&mp_type_OverflowError, "long int not supported in this build"); return mp_const_none; } diff --git a/py/objint_longlong.c b/py/objint_longlong.c index dbfb39809a..b798c91cdf 100644 --- a/py/objint_longlong.c +++ b/py/objint_longlong.c @@ -273,7 +273,7 @@ mp_obj_t mp_obj_new_int_from_float(mp_float_t val) { } #endif -mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base) { +mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { // TODO this does not honor the given length of the string, but it all cases it should anyway be null terminated // TODO check overflow mp_obj_int_t *o = m_new_obj(mp_obj_int_t); diff --git a/py/objint_mpz.c b/py/objint_mpz.c index 2b27df4f63..2cdf225050 100644 --- a/py/objint_mpz.c +++ b/py/objint_mpz.c @@ -407,9 +407,9 @@ mp_obj_t mp_obj_new_int_from_float(mp_float_t val) { } #endif -mp_obj_t mp_obj_new_int_from_str_len(const char **str, mp_uint_t len, bool neg, mp_uint_t base) { +mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base) { mp_obj_int_t *o = mp_obj_int_new_mpz(); - mp_uint_t n = mpz_set_from_str(&o->mpz, *str, len, neg, base); + size_t n = mpz_set_from_str(&o->mpz, *str, len, neg, base); *str += n; return MP_OBJ_FROM_PTR(o); } From 101886f5291fdbef07ba70d386c5e3e644b71cfb Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 16:48:33 +1100 Subject: [PATCH 58/92] py/vm: Convert mp_uint_t to size_t where appropriate. --- py/vm.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/py/vm.c b/py/vm.c index fb59bf9cff..97f344b08e 100644 --- a/py/vm.c +++ b/py/vm.c @@ -63,8 +63,8 @@ typedef enum { do { \ unum = (unum << 7) + (*ip & 0x7f); \ } while ((*ip++ & 0x80) != 0) -#define DECODE_ULABEL mp_uint_t ulab = (ip[0] | (ip[1] << 8)); ip += 2 -#define DECODE_SLABEL mp_uint_t slab = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2 +#define DECODE_ULABEL size_t ulab = (ip[0] | (ip[1] << 8)); ip += 2 +#define DECODE_SLABEL size_t slab = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2 #if MICROPY_PERSISTENT_CODE @@ -863,7 +863,7 @@ unwind_jump:; ENTRY(MP_BC_MAKE_CLOSURE): { DECODE_PTR; - mp_uint_t n_closed_over = *ip++; + size_t n_closed_over = *ip++; // Stack layout: closed_overs <- TOS sp -= n_closed_over - 1; SET_TOP(mp_make_closure_from_raw_code(ptr, n_closed_over, sp)); @@ -872,7 +872,7 @@ unwind_jump:; ENTRY(MP_BC_MAKE_CLOSURE_DEFARGS): { DECODE_PTR; - mp_uint_t n_closed_over = *ip++; + size_t n_closed_over = *ip++; // Stack layout: def_tuple def_dict closed_overs <- TOS sp -= 2 + n_closed_over - 1; SET_TOP(mp_make_closure_from_raw_code(ptr, 0x100 | n_closed_over, sp)); @@ -958,8 +958,8 @@ unwind_jump:; code_state->sp = sp; code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block); - mp_uint_t n_args = unum & 0xff; - mp_uint_t n_kw = (unum >> 8) & 0xff; + size_t n_args = unum & 0xff; + size_t n_kw = (unum >> 8) & 0xff; int adjust = (sp[1] == MP_OBJ_NULL) ? 0 : 1; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(*sp, n_args + adjust, n_kw, sp + 2 - adjust); @@ -1304,7 +1304,7 @@ unwind_loop: // TODO need a better way of not adding traceback to constant objects (right now, just GeneratorExit_obj and MemoryError_obj) if (nlr.ret_val != &mp_const_GeneratorExit_obj && nlr.ret_val != &mp_const_MemoryError_obj) { const byte *ip = code_state->code_info; - mp_uint_t code_info_size = mp_decode_uint(&ip); + size_t code_info_size = mp_decode_uint(&ip); #if MICROPY_PERSISTENT_CODE qstr block_name = ip[0] | (ip[1] << 8); qstr source_file = ip[2] | (ip[3] << 8); @@ -1317,7 +1317,7 @@ unwind_loop: size_t source_line = 1; size_t c; while ((c = *ip)) { - mp_uint_t b, l; + size_t b, l; if ((c & 0x80) == 0) { // 0b0LLBBBBB encoding b = c & 0x1f; From ae8d86758631e62466a55d179897d2111c3cb1c1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 9 Jan 2016 23:14:54 +0000 Subject: [PATCH 59/92] py: Add iter_buf to getiter type method. Allows to iterate over the following without allocating on the heap: - tuple - list - string, bytes - bytearray, array - dict (not dict.keys, dict.values, dict.items) - set, frozenset Allows to call the following without heap memory: - all, any, min, max, sum TODO: still need to allocate stack memory in bytecode for iter_buf. --- cc3200/mods/pybuart.c | 2 +- esp8266/machine_uart.c | 2 +- extmod/modbtree.c | 3 ++- extmod/vfs_fat_file.c | 4 ++-- py/emitnative.c | 1 + py/modbuiltins.c | 14 +++++++++----- py/obj.c | 5 +++++ py/obj.h | 15 +++++++++++++-- py/objarray.c | 14 +++++++++----- py/objdict.c | 24 +++++++++++++++--------- py/objenumerate.c | 6 +++--- py/objfilter.c | 4 ++-- py/objgenerator.c | 2 +- py/objgetitemiter.c | 7 ++++--- py/objlist.c | 14 ++++++++------ py/objmap.c | 4 ++-- py/objpolyiter.c | 2 +- py/objrange.c | 11 ++++++----- py/objreversed.c | 2 +- py/objset.c | 26 +++++++++++++++++--------- py/objstr.c | 17 ++++++++++------- py/objstringio.c | 4 ++-- py/objstrunicode.c | 7 ++++--- py/objtuple.c | 18 +++++++----------- py/objtuple.h | 2 +- py/objtype.c | 4 ++-- py/objzip.c | 4 ++-- py/runtime.c | 26 ++++++++++++++++++-------- py/runtime.h | 2 +- py/vm.c | 2 +- stmhal/pybstdio.c | 4 ++-- stmhal/uart.c | 2 +- stmhal/usb.c | 2 +- unix/file.c | 4 ++-- unix/modffi.c | 6 ++++-- unix/moduselect.c | 2 +- 36 files changed, 162 insertions(+), 106 deletions(-) diff --git a/cc3200/mods/pybuart.c b/cc3200/mods/pybuart.c index fe710be920..dceb842d52 100644 --- a/cc3200/mods/pybuart.c +++ b/cc3200/mods/pybuart.c @@ -664,7 +664,7 @@ const mp_obj_type_t pyb_uart_type = { .name = MP_QSTR_UART, .print = pyb_uart_print, .make_new = pyb_uart_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &uart_stream_p, .locals_dict = (mp_obj_t)&pyb_uart_locals_dict, diff --git a/esp8266/machine_uart.c b/esp8266/machine_uart.c index efdfafd1ae..ef52e8c9a5 100644 --- a/esp8266/machine_uart.c +++ b/esp8266/machine_uart.c @@ -285,7 +285,7 @@ const mp_obj_type_t pyb_uart_type = { .name = MP_QSTR_UART, .print = pyb_uart_print, .make_new = pyb_uart_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &uart_stream_p, .locals_dict = (mp_obj_dict_t*)&pyb_uart_locals_dict, diff --git a/extmod/modbtree.c b/extmod/modbtree.c index bb75845b1b..27f700eea4 100644 --- a/extmod/modbtree.c +++ b/extmod/modbtree.c @@ -184,7 +184,8 @@ STATIC mp_obj_t btree_items(size_t n_args, const mp_obj_t *args) { } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(btree_items_obj, 1, 4, btree_items); -STATIC mp_obj_t btree_getiter(mp_obj_t self_in) { +STATIC mp_obj_t btree_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + (void)iter_buf; mp_obj_btree_t *self = MP_OBJ_TO_PTR(self_in); if (self->next_flags != 0) { // If we're called immediately after keys(), values(), or items(), diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 62da23f948..6263492cbd 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -264,7 +264,7 @@ const mp_obj_type_t mp_type_fileio = { .name = MP_QSTR_FileIO, .print = file_obj_print, .make_new = file_obj_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &fileio_stream_p, .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict, @@ -283,7 +283,7 @@ const mp_obj_type_t mp_type_textio = { .name = MP_QSTR_TextIOWrapper, .print = file_obj_print, .make_new = file_obj_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &textio_stream_p, .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict, diff --git a/py/emitnative.c b/py/emitnative.c index bdb590e77f..e8e3754e17 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1806,6 +1806,7 @@ STATIC void emit_native_get_iter(emit_t *emit) { vtype_kind_t vtype; emit_pre_pop_reg(emit, &vtype, REG_ARG_1); assert(vtype == VTYPE_PYOBJ); + assert(0); // TODO allocate memory for iter_buf emit_call(emit, MP_F_GETITER); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } diff --git a/py/modbuiltins.c b/py/modbuiltins.c index a0c68930d8..13312d2296 100644 --- a/py/modbuiltins.c +++ b/py/modbuiltins.c @@ -117,7 +117,8 @@ STATIC mp_obj_t mp_builtin_abs(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_abs_obj, mp_builtin_abs); STATIC mp_obj_t mp_builtin_all(mp_obj_t o_in) { - mp_obj_t iterable = mp_getiter(o_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(o_in, &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (!mp_obj_is_true(item)) { @@ -129,7 +130,8 @@ STATIC mp_obj_t mp_builtin_all(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_all_obj, mp_builtin_all); STATIC mp_obj_t mp_builtin_any(mp_obj_t o_in) { - mp_obj_t iterable = mp_getiter(o_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(o_in, &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (mp_obj_is_true(item)) { @@ -258,7 +260,7 @@ STATIC mp_obj_t mp_builtin_hex(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_hex_obj, mp_builtin_hex); STATIC mp_obj_t mp_builtin_iter(mp_obj_t o_in) { - return mp_getiter(o_in); + return mp_getiter(o_in, NULL); } MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_iter_obj, mp_builtin_iter); @@ -270,7 +272,8 @@ STATIC mp_obj_t mp_builtin_min_max(size_t n_args, const mp_obj_t *args, mp_map_t mp_obj_t key_fn = key_elem == NULL ? MP_OBJ_NULL : key_elem->value; if (n_args == 1) { // given an iterable - mp_obj_t iterable = mp_getiter(args[0]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(args[0], &iter_buf); mp_obj_t best_key = MP_OBJ_NULL; mp_obj_t best_obj = MP_OBJ_NULL; mp_obj_t item; @@ -495,7 +498,8 @@ STATIC mp_obj_t mp_builtin_sum(size_t n_args, const mp_obj_t *args) { case 1: value = MP_OBJ_NEW_SMALL_INT(0); break; default: value = args[1]; break; } - mp_obj_t iterable = mp_getiter(args[0]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(args[0], &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { value = mp_binary_op(MP_BINARY_OP_ADD, value, item); diff --git a/py/obj.c b/py/obj.c index 1d0c80ab90..4534ef1b28 100644 --- a/py/obj.c +++ b/py/obj.c @@ -481,6 +481,11 @@ mp_obj_t mp_identity(mp_obj_t self) { } MP_DEFINE_CONST_FUN_OBJ_1(mp_identity_obj, mp_identity); +mp_obj_t mp_identity_getiter(mp_obj_t self, mp_obj_iter_buf_t *iter_buf) { + (void)iter_buf; + return self; +} + bool mp_get_buffer(mp_obj_t obj, mp_buffer_info_t *bufinfo, mp_uint_t flags) { mp_obj_type_t *type = mp_obj_get_type(obj); if (type->buffer_p.get_buffer == NULL) { diff --git a/py/obj.h b/py/obj.h index 1169a9095e..79a6d56fad 100644 --- a/py/obj.h +++ b/py/obj.h @@ -417,6 +417,11 @@ typedef enum { PRINT_EXC_SUBCLASS = 0x80, // Internal flag for printing exception subclasses } mp_print_kind_t; +typedef struct _mp_obj_iter_buf_t { + mp_obj_base_t base; + mp_obj_t buf[3]; +} mp_obj_iter_buf_t; + typedef void (*mp_print_fun_t)(const mp_print_t *print, mp_obj_t o, mp_print_kind_t kind); typedef mp_obj_t (*mp_make_new_fun_t)(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args); typedef mp_obj_t (*mp_call_fun_t)(mp_obj_t fun, size_t n_args, size_t n_kw, const mp_obj_t *args); @@ -424,6 +429,7 @@ typedef mp_obj_t (*mp_unary_op_fun_t)(mp_uint_t op, mp_obj_t); typedef mp_obj_t (*mp_binary_op_fun_t)(mp_uint_t op, mp_obj_t, mp_obj_t); typedef void (*mp_attr_fun_t)(mp_obj_t self_in, qstr attr, mp_obj_t *dest); typedef mp_obj_t (*mp_subscr_fun_t)(mp_obj_t self_in, mp_obj_t index, mp_obj_t value); +typedef mp_obj_t (*mp_getiter_fun_t)(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf); // Buffer protocol typedef struct _mp_buffer_info_t { @@ -486,7 +492,11 @@ struct _mp_obj_type_t { // value=MP_OBJ_NULL means delete, value=MP_OBJ_SENTINEL means load, else store // can return MP_OBJ_NULL if op not supported - mp_fun_1_t getiter; // corresponds to __iter__ special method + // corresponds to __iter__ special method + // can use given mp_obj_iter_buf_t to store iterator + // otherwise can return a pointer to an object on the heap + mp_getiter_fun_t getiter; + mp_fun_1_t iternext; // may return MP_OBJ_STOP_ITERATION as an optimisation instead of raising StopIteration() (with no args) mp_buffer_p_t buffer_p; @@ -637,7 +647,7 @@ mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items); mp_obj_t mp_obj_new_slice(mp_obj_t start, mp_obj_t stop, mp_obj_t step); mp_obj_t mp_obj_new_super(mp_obj_t type, mp_obj_t obj); mp_obj_t mp_obj_new_bound_meth(mp_obj_t meth, mp_obj_t self); -mp_obj_t mp_obj_new_getitem_iter(mp_obj_t *args); +mp_obj_t mp_obj_new_getitem_iter(mp_obj_t *args, mp_obj_iter_buf_t *iter_buf); mp_obj_t mp_obj_new_module(qstr module_name); mp_obj_t mp_obj_new_memoryview(byte typecode, size_t nitems, void *items); @@ -775,6 +785,7 @@ qstr mp_obj_code_get_name(const byte *code_info); mp_obj_t mp_identity(mp_obj_t self); MP_DECLARE_CONST_FUN_OBJ_1(mp_identity_obj); +mp_obj_t mp_identity_getiter(mp_obj_t self, mp_obj_iter_buf_t *iter_buf); // module typedef struct _mp_obj_module_t { diff --git a/py/objarray.c b/py/objarray.c index f237918574..c81aebb50c 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -59,7 +59,7 @@ #define TYPECODE_MASK (~(size_t)0) #endif -STATIC mp_obj_t array_iterator_new(mp_obj_t array_in); +STATIC mp_obj_t array_iterator_new(mp_obj_t array_in, mp_obj_iter_buf_t *iter_buf); STATIC mp_obj_t array_append(mp_obj_t self_in, mp_obj_t arg); STATIC mp_obj_t array_extend(mp_obj_t self_in, mp_obj_t arg_in); STATIC mp_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, mp_uint_t flags); @@ -141,7 +141,8 @@ STATIC mp_obj_t array_construct(char typecode, mp_obj_t initializer) { mp_obj_array_t *array = array_new(typecode, len); - mp_obj_t iterable = mp_getiter(initializer); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(initializer, &iter_buf); mp_obj_t item; size_t i = 0; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { @@ -608,15 +609,18 @@ STATIC mp_obj_t array_it_iternext(mp_obj_t self_in) { STATIC const mp_obj_type_t array_it_type = { { &mp_type_type }, .name = MP_QSTR_iterator, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = array_it_iternext, }; -STATIC mp_obj_t array_iterator_new(mp_obj_t array_in) { +STATIC mp_obj_t array_iterator_new(mp_obj_t array_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_array_t) <= sizeof(mp_obj_iter_buf_t)); mp_obj_array_t *array = MP_OBJ_TO_PTR(array_in); - mp_obj_array_it_t *o = m_new0(mp_obj_array_it_t, 1); + mp_obj_array_it_t *o = (mp_obj_array_it_t*)iter_buf; o->base.type = &array_it_type; o->array = array; + o->offset = 0; + o->cur = 0; #if MICROPY_PY_BUILTINS_MEMORYVIEW if (array->base.type == &mp_type_memoryview) { o->offset = array->free; diff --git a/py/objdict.c b/py/objdict.c index 640df7b62a..c52403f715 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -210,8 +210,9 @@ STATIC mp_obj_t dict_it_iternext(mp_obj_t self_in) { } } -STATIC mp_obj_t dict_getiter(mp_obj_t self_in) { - mp_obj_dict_it_t *o = m_new_obj(mp_obj_dict_it_t); +STATIC mp_obj_t dict_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_dict_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_dict_it_t *o = (mp_obj_dict_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = dict_it_iternext; o->dict = self_in; @@ -249,7 +250,8 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(dict_copy_obj, dict_copy); // this is a classmethod STATIC mp_obj_t dict_fromkeys(size_t n_args, const mp_obj_t *args) { - mp_obj_t iter = mp_getiter(args[1]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(args[1], &iter_buf); mp_obj_t value = mp_const_none; mp_obj_t next = MP_OBJ_NULL; @@ -375,10 +377,12 @@ STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwarg } } else { // update from a generic iterable of pairs - mp_obj_t iter = mp_getiter(args[1]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(args[1], &iter_buf); mp_obj_t next = MP_OBJ_NULL; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { - mp_obj_t inneriter = mp_getiter(next); + mp_obj_iter_buf_t inner_iter_buf; + mp_obj_t inneriter = mp_getiter(next, &inner_iter_buf); mp_obj_t key = mp_iternext(inneriter); mp_obj_t value = mp_iternext(inneriter); mp_obj_t stop = mp_iternext(inneriter); @@ -457,14 +461,15 @@ STATIC mp_obj_t dict_view_it_iternext(mp_obj_t self_in) { STATIC const mp_obj_type_t dict_view_it_type = { { &mp_type_type }, .name = MP_QSTR_iterator, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = dict_view_it_iternext, }; -STATIC mp_obj_t dict_view_getiter(mp_obj_t view_in) { +STATIC mp_obj_t dict_view_getiter(mp_obj_t view_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_dict_view_it_t) <= sizeof(mp_obj_iter_buf_t)); mp_check_self(MP_OBJ_IS_TYPE(view_in, &dict_view_type)); mp_obj_dict_view_t *view = MP_OBJ_TO_PTR(view_in); - mp_obj_dict_view_it_t *o = m_new_obj(mp_obj_dict_view_it_t); + mp_obj_dict_view_it_t *o = (mp_obj_dict_view_it_t*)iter_buf; o->base.type = &dict_view_it_type; o->kind = view->kind; o->dict = view->dict; @@ -479,7 +484,8 @@ STATIC void dict_view_print(const mp_print_t *print, mp_obj_t self_in, mp_print_ bool first = true; mp_print_str(print, mp_dict_view_names[self->kind]); mp_print_str(print, "(["); - mp_obj_t self_iter = dict_view_getiter(self_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t self_iter = dict_view_getiter(self_in, &iter_buf); mp_obj_t next = MP_OBJ_NULL; while ((next = dict_view_it_iternext(self_iter)) != MP_OBJ_STOP_ITERATION) { if (!first) { diff --git a/py/objenumerate.c b/py/objenumerate.c index 2b646ca45d..faae6516c0 100644 --- a/py/objenumerate.c +++ b/py/objenumerate.c @@ -56,13 +56,13 @@ STATIC mp_obj_t enumerate_make_new(const mp_obj_type_t *type, size_t n_args, siz // create enumerate object mp_obj_enumerate_t *o = m_new_obj(mp_obj_enumerate_t); o->base.type = type; - o->iter = mp_getiter(arg_vals.iterable.u_obj); + o->iter = mp_getiter(arg_vals.iterable.u_obj, NULL); o->cur = arg_vals.start.u_int; #else (void)n_kw; mp_obj_enumerate_t *o = m_new_obj(mp_obj_enumerate_t); o->base.type = type; - o->iter = mp_getiter(args[0]); + o->iter = mp_getiter(args[0], NULL); o->cur = n_args > 1 ? mp_obj_get_int(args[1]) : 0; #endif @@ -74,7 +74,7 @@ const mp_obj_type_t mp_type_enumerate = { .name = MP_QSTR_enumerate, .make_new = enumerate_make_new, .iternext = enumerate_iternext, - .getiter = mp_identity, + .getiter = mp_identity_getiter, }; STATIC mp_obj_t enumerate_iternext(mp_obj_t self_in) { diff --git a/py/objfilter.c b/py/objfilter.c index a5c85b2cef..a655b8a785 100644 --- a/py/objfilter.c +++ b/py/objfilter.c @@ -39,7 +39,7 @@ STATIC mp_obj_t filter_make_new(const mp_obj_type_t *type, size_t n_args, size_t mp_obj_filter_t *o = m_new_obj(mp_obj_filter_t); o->base.type = type; o->fun = args[0]; - o->iter = mp_getiter(args[1]); + o->iter = mp_getiter(args[1], NULL); return MP_OBJ_FROM_PTR(o); } @@ -65,7 +65,7 @@ const mp_obj_type_t mp_type_filter = { { &mp_type_type }, .name = MP_QSTR_filter, .make_new = filter_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = filter_iternext, }; diff --git a/py/objgenerator.c b/py/objgenerator.c index 0be9b8d59c..654b186703 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -236,7 +236,7 @@ const mp_obj_type_t mp_type_gen_instance = { { &mp_type_type }, .name = MP_QSTR_generator, .print = gen_instance_print, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = gen_instance_iternext, .locals_dict = (mp_obj_dict_t*)&gen_instance_locals_dict, }; diff --git a/py/objgetitemiter.c b/py/objgetitemiter.c index 526180fdbc..a3c754448f 100644 --- a/py/objgetitemiter.c +++ b/py/objgetitemiter.c @@ -61,13 +61,14 @@ STATIC mp_obj_t it_iternext(mp_obj_t self_in) { STATIC const mp_obj_type_t it_type = { { &mp_type_type }, .name = MP_QSTR_iterator, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = it_iternext, }; // args are those returned from mp_load_method_maybe (ie either an attribute or a method) -mp_obj_t mp_obj_new_getitem_iter(mp_obj_t *args) { - mp_obj_getitem_iter_t *o = m_new_obj(mp_obj_getitem_iter_t); +mp_obj_t mp_obj_new_getitem_iter(mp_obj_t *args, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_getitem_iter_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_getitem_iter_t *o = (mp_obj_getitem_iter_t*)iter_buf; o->base.type = &it_type; o->args[0] = args[0]; o->args[1] = args[1]; diff --git a/py/objlist.c b/py/objlist.c index 210140388a..28da109912 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -33,7 +33,7 @@ #include "py/runtime.h" #include "py/stackctrl.h" -STATIC mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur); +STATIC mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur, mp_obj_iter_buf_t *iter_buf); STATIC mp_obj_list_t *list_new(size_t n); STATIC mp_obj_t list_extend(mp_obj_t self_in, mp_obj_t arg_in); STATIC mp_obj_t list_pop(size_t n_args, const mp_obj_t *args); @@ -60,7 +60,8 @@ STATIC void list_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t k } STATIC mp_obj_t list_extend_from_iter(mp_obj_t list, mp_obj_t iterable) { - mp_obj_t iter = mp_getiter(iterable); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(iterable, &iter_buf); mp_obj_t item; while ((item = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { mp_obj_list_append(list, item); @@ -225,8 +226,8 @@ STATIC mp_obj_t list_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } } -STATIC mp_obj_t list_getiter(mp_obj_t o_in) { - return mp_obj_new_list_iterator(o_in, 0); +STATIC mp_obj_t list_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { + return mp_obj_new_list_iterator(o_in, 0, iter_buf); } mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg) { @@ -516,8 +517,9 @@ STATIC mp_obj_t list_it_iternext(mp_obj_t self_in) { } } -mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur) { - mp_obj_list_it_t *o = m_new_obj(mp_obj_list_it_t); +mp_obj_t mp_obj_new_list_iterator(mp_obj_t list, size_t cur, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_list_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_list_it_t *o = (mp_obj_list_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = list_it_iternext; o->list = list; diff --git a/py/objmap.c b/py/objmap.c index ed0291435d..0b2189016e 100644 --- a/py/objmap.c +++ b/py/objmap.c @@ -43,7 +43,7 @@ STATIC mp_obj_t map_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ o->n_iters = n_args - 1; o->fun = args[0]; for (mp_uint_t i = 0; i < n_args - 1; i++) { - o->iters[i] = mp_getiter(args[i + 1]); + o->iters[i] = mp_getiter(args[i + 1], NULL); } return MP_OBJ_FROM_PTR(o); } @@ -68,6 +68,6 @@ const mp_obj_type_t mp_type_map = { { &mp_type_type }, .name = MP_QSTR_map, .make_new = map_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = map_iternext, }; diff --git a/py/objpolyiter.c b/py/objpolyiter.c index 9bba538c5f..61bd1e0ac2 100644 --- a/py/objpolyiter.c +++ b/py/objpolyiter.c @@ -49,6 +49,6 @@ STATIC mp_obj_t polymorph_it_iternext(mp_obj_t self_in) { const mp_obj_type_t mp_type_polymorph_iter = { { &mp_type_type }, .name = MP_QSTR_iterator, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = polymorph_it_iternext, }; diff --git a/py/objrange.c b/py/objrange.c index 79459316b1..dd074a98ad 100644 --- a/py/objrange.c +++ b/py/objrange.c @@ -55,12 +55,13 @@ STATIC mp_obj_t range_it_iternext(mp_obj_t o_in) { STATIC const mp_obj_type_t range_it_type = { { &mp_type_type }, .name = MP_QSTR_iterator, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = range_it_iternext, }; -STATIC mp_obj_t mp_obj_new_range_iterator(mp_int_t cur, mp_int_t stop, mp_int_t step) { - mp_obj_range_it_t *o = m_new_obj(mp_obj_range_it_t); +STATIC mp_obj_t mp_obj_new_range_iterator(mp_int_t cur, mp_int_t stop, mp_int_t step, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_range_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_range_it_t *o = (mp_obj_range_it_t*)iter_buf; o->base.type = &range_it_type; o->cur = cur; o->stop = stop; @@ -161,9 +162,9 @@ STATIC mp_obj_t range_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } } -STATIC mp_obj_t range_getiter(mp_obj_t o_in) { +STATIC mp_obj_t range_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { mp_obj_range_t *o = MP_OBJ_TO_PTR(o_in); - return mp_obj_new_range_iterator(o->start, o->stop, o->step); + return mp_obj_new_range_iterator(o->start, o->stop, o->step, iter_buf); } diff --git a/py/objreversed.c b/py/objreversed.c index 4343c19780..fc85e72bfb 100644 --- a/py/objreversed.c +++ b/py/objreversed.c @@ -74,7 +74,7 @@ const mp_obj_type_t mp_type_reversed = { { &mp_type_type }, .name = MP_QSTR_reversed, .make_new = reversed_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = reversed_iternext, }; diff --git a/py/objset.c b/py/objset.c index b9da44a44f..99e1e8ca57 100644 --- a/py/objset.c +++ b/py/objset.c @@ -129,7 +129,8 @@ STATIC mp_obj_t set_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ default: { // can only be 0 or 1 arg // 1 argument, an iterable from which we make a new set mp_obj_t set = mp_obj_new_set(0, NULL); - mp_obj_t iterable = mp_getiter(args[0]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(args[0], &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { mp_obj_set_store(set, item); @@ -156,8 +157,9 @@ STATIC mp_obj_t set_it_iternext(mp_obj_t self_in) { return MP_OBJ_STOP_ITERATION; } -STATIC mp_obj_t set_getiter(mp_obj_t set_in) { - mp_obj_set_it_t *o = m_new_obj(mp_obj_set_it_t); +STATIC mp_obj_t set_getiter(mp_obj_t set_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_set_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_set_it_t *o = (mp_obj_set_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = set_it_iternext; o->set = (mp_obj_set_t *)MP_OBJ_TO_PTR(set_in); @@ -233,7 +235,8 @@ STATIC mp_obj_t set_diff_int(size_t n_args, const mp_obj_t *args, bool update) { if (self == other) { set_clear(self); } else { - mp_obj_t iter = mp_getiter(other); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(other, &iter_buf); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { set_discard(self, next); @@ -270,7 +273,8 @@ STATIC mp_obj_t set_intersect_int(mp_obj_t self_in, mp_obj_t other, bool update) mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); mp_obj_set_t *out = MP_OBJ_TO_PTR(mp_obj_new_set(0, NULL)); - mp_obj_t iter = mp_getiter(other); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(other, &iter_buf); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { if (mp_set_lookup(&self->set, next, MP_MAP_LOOKUP)) { @@ -302,7 +306,8 @@ STATIC mp_obj_t set_isdisjoint(mp_obj_t self_in, mp_obj_t other) { check_set_or_frozenset(self_in); mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); - mp_obj_t iter = mp_getiter(other); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(other, &iter_buf); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { if (mp_set_lookup(&self->set, next, MP_MAP_LOOKUP)) { @@ -335,7 +340,8 @@ STATIC mp_obj_t set_issubset_internal(mp_obj_t self_in, mp_obj_t other_in, bool if (proper && self->set.used == other->set.used) { out = false; } else { - mp_obj_t iter = set_getiter(MP_OBJ_FROM_PTR(self)); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = set_getiter(MP_OBJ_FROM_PTR(self), &iter_buf); mp_obj_t next; while ((next = set_it_iternext(iter)) != MP_OBJ_STOP_ITERATION) { if (!mp_set_lookup(&other->set, next, MP_MAP_LOOKUP)) { @@ -408,7 +414,8 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(set_remove_obj, set_remove); STATIC mp_obj_t set_symmetric_difference_update(mp_obj_t self_in, mp_obj_t other_in) { check_set(self_in); mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); - mp_obj_t iter = mp_getiter(other_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(other_in, &iter_buf); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { mp_set_lookup(&self->set, next, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND_OR_REMOVE_IF_FOUND); @@ -427,7 +434,8 @@ STATIC mp_obj_t set_symmetric_difference(mp_obj_t self_in, mp_obj_t other_in) { STATIC MP_DEFINE_CONST_FUN_OBJ_2(set_symmetric_difference_obj, set_symmetric_difference); STATIC void set_update_int(mp_obj_set_t *self, mp_obj_t other_in) { - mp_obj_t iter = mp_getiter(other_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(other_in, &iter_buf); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { mp_set_lookup(&self->set, next, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); diff --git a/py/objstr.c b/py/objstr.c index 262b89ddd8..c137afe673 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -38,7 +38,7 @@ STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, size_t n_args, const mp_obj_t *args, mp_obj_t dict); -STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); +STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf); STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); /******************************************************************************/ @@ -231,7 +231,8 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size vstr_init(&vstr, len); } - mp_obj_t iterable = mp_getiter(args[0]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(args[0], &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { mp_int_t val = mp_obj_get_int(item); @@ -1942,7 +1943,7 @@ STATIC const mp_rom_map_elem_t str8_locals_dict_table[] = { STATIC MP_DEFINE_CONST_DICT(str8_locals_dict, str8_locals_dict_table); #if !MICROPY_PY_BUILTINS_STR_UNICODE -STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf); const mp_obj_type_t mp_type_str = { { &mp_type_type }, @@ -2142,8 +2143,9 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { } } -STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { - mp_obj_str8_it_t *o = m_new_obj(mp_obj_str8_it_t); +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_str8_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_str8_it_t *o = (mp_obj_str8_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = str_it_iternext; o->str = str; @@ -2164,8 +2166,9 @@ STATIC mp_obj_t bytes_it_iternext(mp_obj_t self_in) { } } -mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str) { - mp_obj_str8_it_t *o = m_new_obj(mp_obj_str8_it_t); +mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_str8_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_str8_it_t *o = (mp_obj_str8_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = bytes_it_iternext; o->str = str; diff --git a/py/objstringio.c b/py/objstringio.c index a77ffae246..61a30752ed 100644 --- a/py/objstringio.c +++ b/py/objstringio.c @@ -215,7 +215,7 @@ const mp_obj_type_t mp_type_stringio = { .name = MP_QSTR_StringIO, .print = stringio_print, .make_new = stringio_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &stringio_stream_p, .locals_dict = (mp_obj_dict_t*)&stringio_locals_dict, @@ -227,7 +227,7 @@ const mp_obj_type_t mp_type_bytesio = { .name = MP_QSTR_BytesIO, .print = stringio_print, .make_new = stringio_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &bytesio_stream_p, .locals_dict = (mp_obj_dict_t*)&stringio_locals_dict, diff --git a/py/objstrunicode.c b/py/objstrunicode.c index 0091edd725..441ec293d9 100644 --- a/py/objstrunicode.c +++ b/py/objstrunicode.c @@ -36,7 +36,7 @@ #if MICROPY_PY_BUILTINS_STR_UNICODE -STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf); /******************************************************************************/ /* str */ @@ -301,8 +301,9 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { } } -STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str) { - mp_obj_str_it_t *o = m_new_obj(mp_obj_str_it_t); +STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_str_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_str_it_t *o = (mp_obj_str_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = str_it_iternext; o->str = str; diff --git a/py/objtuple.c b/py/objtuple.c index 1935a67098..ad45696cae 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -32,8 +32,6 @@ #include "py/runtime0.h" #include "py/runtime.h" -STATIC mp_obj_t mp_obj_new_tuple_iterator(mp_obj_tuple_t *tuple, size_t cur); - /******************************************************************************/ /* tuple */ @@ -84,7 +82,8 @@ STATIC mp_obj_t mp_obj_tuple_make_new(const mp_obj_type_t *type_in, size_t n_arg size_t len = 0; mp_obj_t *items = m_new(mp_obj_t, alloc); - mp_obj_t iterable = mp_getiter(args[0]); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(args[0], &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (len >= alloc) { @@ -195,10 +194,6 @@ mp_obj_t mp_obj_tuple_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } } -mp_obj_t mp_obj_tuple_getiter(mp_obj_t o_in) { - return mp_obj_new_tuple_iterator(MP_OBJ_TO_PTR(o_in), 0); -} - STATIC mp_obj_t tuple_count(mp_obj_t self_in, mp_obj_t value) { mp_check_self(MP_OBJ_IS_TYPE(self_in, &mp_type_tuple)); mp_obj_tuple_t *self = MP_OBJ_TO_PTR(self_in); @@ -284,11 +279,12 @@ STATIC mp_obj_t tuple_it_iternext(mp_obj_t self_in) { } } -STATIC mp_obj_t mp_obj_new_tuple_iterator(mp_obj_tuple_t *tuple, size_t cur) { - mp_obj_tuple_it_t *o = m_new_obj(mp_obj_tuple_it_t); +mp_obj_t mp_obj_tuple_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_tuple_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_obj_tuple_it_t *o = (mp_obj_tuple_it_t*)iter_buf; o->base.type = &mp_type_polymorph_iter; o->iternext = tuple_it_iternext; - o->tuple = tuple; - o->cur = cur; + o->tuple = MP_OBJ_TO_PTR(o_in); + o->cur = 0; return MP_OBJ_FROM_PTR(o); } diff --git a/py/objtuple.h b/py/objtuple.h index 760135f866..555c3b3c2c 100644 --- a/py/objtuple.h +++ b/py/objtuple.h @@ -44,7 +44,7 @@ void mp_obj_tuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_obj_t mp_obj_tuple_unary_op(mp_uint_t op, mp_obj_t self_in); mp_obj_t mp_obj_tuple_binary_op(mp_uint_t op, mp_obj_t lhs, mp_obj_t rhs); mp_obj_t mp_obj_tuple_subscr(mp_obj_t base, mp_obj_t index, mp_obj_t value); -mp_obj_t mp_obj_tuple_getiter(mp_obj_t o_in); +mp_obj_t mp_obj_tuple_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf); extern const mp_obj_type_t mp_type_attrtuple; diff --git a/py/objtype.c b/py/objtype.c index 60f630c3dc..a4c424929d 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -758,7 +758,7 @@ mp_obj_t mp_obj_instance_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons return mp_call_method_self_n_kw(member[0], member[1], n_args, n_kw, args); } -STATIC mp_obj_t instance_getiter(mp_obj_t self_in) { +STATIC mp_obj_t instance_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { mp_obj_instance_t *self = MP_OBJ_TO_PTR(self_in); mp_obj_t member[2] = {MP_OBJ_NULL}; struct class_lookup_data lookup = { @@ -773,7 +773,7 @@ STATIC mp_obj_t instance_getiter(mp_obj_t self_in) { return MP_OBJ_NULL; } else if (member[0] == MP_OBJ_SENTINEL) { mp_obj_type_t *type = mp_obj_get_type(self->subobj[0]); - return type->getiter(self->subobj[0]); + return type->getiter(self->subobj[0], iter_buf); } else { return mp_call_method_n_kw(0, 0, member); } diff --git a/py/objzip.c b/py/objzip.c index 6edefc3611..5d9ba48e78 100644 --- a/py/objzip.c +++ b/py/objzip.c @@ -43,7 +43,7 @@ STATIC mp_obj_t zip_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ o->base.type = type; o->n_iters = n_args; for (mp_uint_t i = 0; i < n_args; i++) { - o->iters[i] = mp_getiter(args[i]); + o->iters[i] = mp_getiter(args[i], NULL); } return MP_OBJ_FROM_PTR(o); } @@ -71,6 +71,6 @@ const mp_obj_type_t mp_type_zip = { { &mp_type_type }, .name = MP_QSTR_zip, .make_new = zip_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = zip_iternext, }; diff --git a/py/runtime.c b/py/runtime.c index 3ac30f58e3..b9d7b72dcf 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -520,7 +520,8 @@ mp_obj_t mp_binary_op(mp_uint_t op, mp_obj_t lhs, mp_obj_t rhs) { } if (type->getiter != NULL) { /* second attempt, walk the iterator */ - mp_obj_t iter = mp_getiter(rhs); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(rhs, &iter_buf); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { if (mp_obj_equal(next, lhs)) { @@ -698,7 +699,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ args2_len += n_args; // extract the variable position args from the iterator - mp_obj_t iterable = mp_getiter(pos_seq); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(pos_seq, &iter_buf); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (args2_len >= args2_alloc) { @@ -743,7 +745,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // get the keys iterable mp_obj_t dest[3]; mp_load_method(kw_dict, MP_QSTR_keys, dest); - mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest)); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest), &iter_buf); mp_obj_t key; while ((key = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { @@ -809,7 +812,8 @@ void mp_unpack_sequence(mp_obj_t seq_in, size_t num, mp_obj_t *items) { items[i] = seq_items[num - 1 - i]; } } else { - mp_obj_t iterable = mp_getiter(seq_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(seq_in, &iter_buf); for (seq_len = 0; seq_len < num; seq_len++) { mp_obj_t el = mp_iternext(iterable); @@ -873,7 +877,8 @@ void mp_unpack_ex(mp_obj_t seq_in, size_t num_in, mp_obj_t *items) { // items destination array, then the rest to a dynamically created list. Once the // iterable is exhausted, we take from this list for the right part of the items. // TODO Improve to waste less memory in the dynamically created list. - mp_obj_t iterable = mp_getiter(seq_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(seq_in, &iter_buf); mp_obj_t item; for (seq_len = 0; seq_len < num_left; seq_len++) { item = mp_iternext(iterable); @@ -1096,13 +1101,18 @@ void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t value) { } } -mp_obj_t mp_getiter(mp_obj_t o_in) { +mp_obj_t mp_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { assert(o_in); + // if caller did not provide a buffer then allocate one on the heap + if (iter_buf == NULL) { + iter_buf = m_new_obj(mp_obj_iter_buf_t); + } + // check for native getiter (corresponds to __iter__) mp_obj_type_t *type = mp_obj_get_type(o_in); if (type->getiter != NULL) { - mp_obj_t iter = type->getiter(o_in); + mp_obj_t iter = type->getiter(o_in, iter_buf); if (iter != MP_OBJ_NULL) { return iter; } @@ -1113,7 +1123,7 @@ mp_obj_t mp_getiter(mp_obj_t o_in) { mp_load_method_maybe(o_in, MP_QSTR___getitem__, dest); if (dest[0] != MP_OBJ_NULL) { // __getitem__ exists, create and return an iterator - return mp_obj_new_getitem_iter(dest); + return mp_obj_new_getitem_iter(dest, iter_buf); } // object not iterable diff --git a/py/runtime.h b/py/runtime.h index e25f2a483d..954833b67a 100644 --- a/py/runtime.h +++ b/py/runtime.h @@ -123,7 +123,7 @@ void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest); void mp_load_method_maybe(mp_obj_t base, qstr attr, mp_obj_t *dest); void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t val); -mp_obj_t mp_getiter(mp_obj_t o); +mp_obj_t mp_getiter(mp_obj_t o, mp_obj_iter_buf_t *iter_buf); mp_obj_t mp_iternext_allow_raise(mp_obj_t o); // may return MP_OBJ_STOP_ITERATION instead of raising StopIteration() mp_obj_t mp_iternext(mp_obj_t o); // will always return MP_OBJ_STOP_ITERATION instead of raising StopIteration(...) mp_vm_return_kind_t mp_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val); diff --git a/py/vm.c b/py/vm.c index 97f344b08e..917d41a11b 100644 --- a/py/vm.c +++ b/py/vm.c @@ -723,7 +723,7 @@ unwind_jump:; ENTRY(MP_BC_GET_ITER): MARK_EXC_IP_SELECTIVE(); - SET_TOP(mp_getiter(TOP())); + SET_TOP(mp_getiter(TOP(), NULL)); DISPATCH(); ENTRY(MP_BC_FOR_ITER): { diff --git a/stmhal/pybstdio.c b/stmhal/pybstdio.c index dec4f227ad..9b1bfff906 100644 --- a/stmhal/pybstdio.c +++ b/stmhal/pybstdio.c @@ -120,7 +120,7 @@ STATIC const mp_obj_type_t stdio_obj_type = { .name = MP_QSTR_FileIO, // TODO .make_new? .print = stdio_obj_print, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &stdio_obj_stream_p, .locals_dict = (mp_obj_t)&stdio_locals_dict, @@ -153,7 +153,7 @@ STATIC const mp_obj_type_t stdio_buffer_obj_type = { { &mp_type_type }, .name = MP_QSTR_FileIO, .print = stdio_obj_print, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &stdio_buffer_obj_stream_p, .locals_dict = (mp_obj_t)&stdio_locals_dict, diff --git a/stmhal/uart.c b/stmhal/uart.c index d53ca5b80c..4fae3a80c4 100644 --- a/stmhal/uart.c +++ b/stmhal/uart.c @@ -1037,7 +1037,7 @@ const mp_obj_type_t pyb_uart_type = { .name = MP_QSTR_UART, .print = pyb_uart_print, .make_new = pyb_uart_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &uart_stream_p, .locals_dict = (mp_obj_t)&pyb_uart_locals_dict, diff --git a/stmhal/usb.c b/stmhal/usb.c index c413ce4bac..f71c70665e 100644 --- a/stmhal/usb.c +++ b/stmhal/usb.c @@ -518,7 +518,7 @@ const mp_obj_type_t pyb_usb_vcp_type = { .name = MP_QSTR_USB_VCP, .print = pyb_usb_vcp_print, .make_new = pyb_usb_vcp_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &pyb_usb_vcp_stream_p, .locals_dict = (mp_obj_t)&pyb_usb_vcp_locals_dict, diff --git a/unix/file.c b/unix/file.c index 96fe78c494..a60840c813 100644 --- a/unix/file.c +++ b/unix/file.c @@ -244,7 +244,7 @@ const mp_obj_type_t mp_type_fileio = { .name = MP_QSTR_FileIO, .print = fdfile_print, .make_new = fdfile_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &fileio_stream_p, .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict, @@ -263,7 +263,7 @@ const mp_obj_type_t mp_type_textio = { .name = MP_QSTR_TextIOWrapper, .print = fdfile_print, .make_new = fdfile_make_new, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = mp_stream_unbuffered_iter, .protocol = &textio_stream_p, .locals_dict = (mp_obj_dict_t*)&rawfile_locals_dict, diff --git a/unix/modffi.c b/unix/modffi.c index cae16c579a..74194e2cc0 100644 --- a/unix/modffi.c +++ b/unix/modffi.c @@ -194,7 +194,8 @@ STATIC mp_obj_t make_func(mp_obj_t rettype_in, void *func, mp_obj_t argtypes_in) o->rettype = *rettype; o->argtypes = argtypes; - mp_obj_t iterable = mp_getiter(argtypes_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(argtypes_in, &iter_buf); mp_obj_t item; int i = 0; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { @@ -251,7 +252,8 @@ STATIC mp_obj_t mod_ffi_callback(mp_obj_t rettype_in, mp_obj_t func_in, mp_obj_t o->rettype = *rettype; - mp_obj_t iterable = mp_getiter(paramtypes_in); + mp_obj_iter_buf_t iter_buf; + mp_obj_t iterable = mp_getiter(paramtypes_in, &iter_buf); mp_obj_t item; int i = 0; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { diff --git a/unix/moduselect.c b/unix/moduselect.c index f966efa412..76938329fc 100644 --- a/unix/moduselect.c +++ b/unix/moduselect.c @@ -288,7 +288,7 @@ STATIC MP_DEFINE_CONST_DICT(poll_locals_dict, poll_locals_dict_table); STATIC const mp_obj_type_t mp_type_poll = { { &mp_type_type }, .name = MP_QSTR_poll, - .getiter = mp_identity, + .getiter = mp_identity_getiter, .iternext = poll_iternext, .locals_dict = (void*)&poll_locals_dict, }; From f4df3aaa72a0460614b1ab8b7b8a7927a1165e31 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 9 Jan 2016 23:59:52 +0000 Subject: [PATCH 60/92] py: Allow bytecode/native to put iter_buf on stack for simple for loops. So that the "for x in it: ..." statement can now work without using the heap (so long as the iterator argument fits in an iter_buf structure). --- py/bc0.h | 1 + py/compile.c | 12 ++++++------ py/emit.h | 8 ++++---- py/emitbc.c | 17 +++++++++++++---- py/emitnative.c | 14 +++++++++++--- py/showbc.c | 4 ++++ py/vm.c | 11 +++++++++++ py/vmentrytable.h | 1 + tests/cmdline/cmd_showbc.py.exp | 8 ++++++-- 9 files changed, 57 insertions(+), 19 deletions(-) diff --git a/py/bc0.h b/py/bc0.h index 5ff9e50a89..c2b019f1a9 100644 --- a/py/bc0.h +++ b/py/bc0.h @@ -79,6 +79,7 @@ #define MP_BC_POP_BLOCK (0x44) #define MP_BC_POP_EXCEPT (0x45) #define MP_BC_UNWIND_JUMP (0x46) // rel byte code offset, 16-bit signed, in excess; then a byte +#define MP_BC_GET_ITER_STACK (0x47) #define MP_BC_BUILD_TUPLE (0x50) // uint #define MP_BC_BUILD_LIST (0x51) // uint diff --git a/py/compile.c b/py/compile.c index b84793d10a..d2fa03f01e 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1475,7 +1475,7 @@ STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { uint pop_label = comp_next_label(comp); compile_node(comp, pns->nodes[1]); // iterator - EMIT(get_iter); + EMIT_ARG(get_iter, true); EMIT_ARG(label_assign, continue_label); EMIT_ARG(for_iter, pop_label); c_assign(comp, pns->nodes[0], ASSIGN_STORE); // variable @@ -1484,7 +1484,7 @@ STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(jump, continue_label); } EMIT_ARG(label_assign, pop_label); - EMIT(for_iter_end); + EMIT_ARG(for_iter_end, true); // break/continue apply to outer loop (if any) in the else block END_BREAK_CONTINUE_BLOCK @@ -1680,7 +1680,7 @@ STATIC void compile_with_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { } STATIC void compile_yield_from(compiler_t *comp) { - EMIT(get_iter); + EMIT_ARG(get_iter, false); EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE); EMIT(yield_from); } @@ -2372,7 +2372,7 @@ STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, close_over_variables_etc(comp, this_scope, 0, 0); compile_node(comp, pns_comp_for->nodes[1]); // source of the iterator - EMIT(get_iter); + EMIT_ARG(get_iter, false); EMIT_ARG(call_function, 1, 0, 0); } @@ -2900,13 +2900,13 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn // for loop mp_parse_node_struct_t *pns_comp_for2 = (mp_parse_node_struct_t*)pn_iter; compile_node(comp, pns_comp_for2->nodes[1]); - EMIT(get_iter); + EMIT_ARG(get_iter, false); compile_scope_comp_iter(comp, pns_comp_for2, pn_inner_expr, for_depth + 1); } EMIT_ARG(jump, l_top); EMIT_ARG(label_assign, l_end); - EMIT(for_iter_end); + EMIT_ARG(for_iter_end, false); } STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { diff --git a/py/emit.h b/py/emit.h index 7d00f11f95..6acae9e80f 100644 --- a/py/emit.h +++ b/py/emit.h @@ -110,9 +110,9 @@ typedef struct _emit_method_table_t { void (*setup_except)(emit_t *emit, mp_uint_t label); void (*setup_finally)(emit_t *emit, mp_uint_t label); void (*end_finally)(emit_t *emit); - void (*get_iter)(emit_t *emit); + void (*get_iter)(emit_t *emit, bool use_stack); void (*for_iter)(emit_t *emit, mp_uint_t label); - void (*for_iter_end)(emit_t *emit); + void (*for_iter_end)(emit_t *emit, bool use_stack); void (*pop_block)(emit_t *emit); void (*pop_except)(emit_t *emit); void (*unary_op)(emit_t *emit, mp_unary_op_t op); @@ -228,9 +228,9 @@ void mp_emit_bc_with_cleanup(emit_t *emit, mp_uint_t label); void mp_emit_bc_setup_except(emit_t *emit, mp_uint_t label); void mp_emit_bc_setup_finally(emit_t *emit, mp_uint_t label); void mp_emit_bc_end_finally(emit_t *emit); -void mp_emit_bc_get_iter(emit_t *emit); +void mp_emit_bc_get_iter(emit_t *emit, bool use_stack); void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label); -void mp_emit_bc_for_iter_end(emit_t *emit); +void mp_emit_bc_for_iter_end(emit_t *emit, bool use_stack); void mp_emit_bc_pop_block(emit_t *emit); void mp_emit_bc_pop_except(emit_t *emit); void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op); diff --git a/py/emitbc.c b/py/emitbc.c index e3a047f272..3e0c0b3972 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -734,6 +734,10 @@ void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_dept if (label & MP_EMIT_BREAK_FROM_FOR) { // need to pop the iterator if we are breaking out of a for loop emit_write_bytecode_byte(emit, MP_BC_POP_TOP); + // also pop the iter_buf + for (size_t i = 0; i < sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); ++i) { + emit_write_bytecode_byte(emit, MP_BC_POP_TOP); + } } emit_write_bytecode_byte_signed_label(emit, MP_BC_JUMP, label & ~MP_EMIT_BREAK_FROM_FOR); } else { @@ -773,9 +777,9 @@ void mp_emit_bc_end_finally(emit_t *emit) { emit_write_bytecode_byte(emit, MP_BC_END_FINALLY); } -void mp_emit_bc_get_iter(emit_t *emit) { - emit_bc_pre(emit, 0); - emit_write_bytecode_byte(emit, MP_BC_GET_ITER); +void mp_emit_bc_get_iter(emit_t *emit, bool use_stack) { + emit_bc_pre(emit, use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : 0); + emit_write_bytecode_byte(emit, use_stack ? MP_BC_GET_ITER_STACK : MP_BC_GET_ITER); } void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) { @@ -783,8 +787,13 @@ void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) { emit_write_bytecode_byte_unsigned_label(emit, MP_BC_FOR_ITER, label); } -void mp_emit_bc_for_iter_end(emit_t *emit) { +void mp_emit_bc_for_iter_end(emit_t *emit, bool use_stack) { emit_bc_pre(emit, -1); + if (use_stack) { + for (size_t i = 0; i < sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); ++i) { + mp_emit_bc_pop_top(emit); + } + } } void mp_emit_bc_pop_block(emit_t *emit) { diff --git a/py/emitnative.c b/py/emitnative.c index e8e3754e17..6e8a37bdea 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1799,14 +1799,19 @@ STATIC void emit_native_end_finally(emit_t *emit) { emit_post(emit); } -STATIC void emit_native_get_iter(emit_t *emit) { +STATIC void emit_native_get_iter(emit_t *emit, bool use_stack) { // perhaps the difficult one, as we want to rewrite for loops using native code // in cases where we iterate over a Python object, can we use normal runtime calls? vtype_kind_t vtype; emit_pre_pop_reg(emit, &vtype, REG_ARG_1); assert(vtype == VTYPE_PYOBJ); - assert(0); // TODO allocate memory for iter_buf + if (use_stack) { + emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_2, sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t)); + } else { + // mp_getiter will allocate the iter_buf on the heap + ASM_MOV_IMM_TO_REG(emit->as, 0, REG_ARG_2); + } emit_call(emit, MP_F_GETITER); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -1822,10 +1827,13 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) { emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } -STATIC void emit_native_for_iter_end(emit_t *emit) { +STATIC void emit_native_for_iter_end(emit_t *emit, bool use_stack) { // adjust stack counter (we get here from for_iter ending, which popped the value for us) emit_native_pre(emit); adjust_stack(emit, -1); + if (use_stack) { + adjust_stack(emit, -(sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t))); + } emit_post(emit); } diff --git a/py/showbc.c b/py/showbc.c index 9d20decdc8..b52905f677 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -387,6 +387,10 @@ const byte *mp_bytecode_print_str(const byte *ip) { printf("GET_ITER"); break; + case MP_BC_GET_ITER_STACK: + printf("GET_ITER_STACK"); + break; + case MP_BC_FOR_ITER: DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward printf("FOR_ITER " UINT_FMT, (mp_uint_t)(ip + unum - mp_showbc_code_start)); diff --git a/py/vm.c b/py/vm.c index 917d41a11b..848a77a453 100644 --- a/py/vm.c +++ b/py/vm.c @@ -681,7 +681,9 @@ unwind_jump:; } ip = (const byte*)MP_OBJ_TO_PTR(POP()); // pop destination ip for jump if (unum != 0) { + // pop iter and iter_buf sp--; + sp -= sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); } DISPATCH_WITH_PEND_EXC_CHECK(); } @@ -726,6 +728,15 @@ unwind_jump:; SET_TOP(mp_getiter(TOP(), NULL)); DISPATCH(); + ENTRY(MP_BC_GET_ITER_STACK): { + MARK_EXC_IP_SELECTIVE(); + mp_obj_t obj = TOP(); + mp_obj_iter_buf_t *iter_buf = (mp_obj_iter_buf_t*)sp; + sp += sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); + SET_TOP(mp_getiter(obj, iter_buf)); + DISPATCH(); + } + ENTRY(MP_BC_FOR_ITER): { MARK_EXC_IP_SELECTIVE(); DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward diff --git a/py/vmentrytable.h b/py/vmentrytable.h index dd30dd7a54..8731c3d4c4 100644 --- a/py/vmentrytable.h +++ b/py/vmentrytable.h @@ -73,6 +73,7 @@ static const void *const entry_table[256] = { [MP_BC_SETUP_FINALLY] = &&entry_MP_BC_SETUP_FINALLY, [MP_BC_END_FINALLY] = &&entry_MP_BC_END_FINALLY, [MP_BC_GET_ITER] = &&entry_MP_BC_GET_ITER, + [MP_BC_GET_ITER_STACK] = &&entry_MP_BC_GET_ITER_STACK, [MP_BC_FOR_ITER] = &&entry_MP_BC_FOR_ITER, [MP_BC_POP_BLOCK] = &&entry_MP_BC_POP_BLOCK, [MP_BC_POP_EXCEPT] = &&entry_MP_BC_POP_EXCEPT, diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 1f0c054cf9..8ee47eab66 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -31,7 +31,7 @@ File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ byt Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## \.\+rg names: -(N_STATE 22) +(N_STATE 23) (N_EXC_STACK 2) (INIT_CELL 14) (INIT_CELL 15) @@ -245,12 +245,16 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): \\d\+ LOAD_FAST 0 \\d\+ STORE_FAST 0 \\d\+ LOAD_DEREF 14 -\\d\+ GET_ITER +\\d\+ GET_ITER_STACK \\d\+ FOR_ITER \\d\+ \\d\+ STORE_FAST 0 \\d\+ LOAD_FAST 1 \\d\+ POP_TOP \\d\+ JUMP \\d\+ +\\d\+ POP_TOP +\\d\+ POP_TOP +\\d\+ POP_TOP +\\d\+ POP_TOP \\d\+ SETUP_FINALLY \\d\+ \\d\+ SETUP_EXCEPT \\d\+ \\d\+ JUMP \\d\+ From 6e769da0da2ae24cbdab50c57f0d088e137146e3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 17 Jan 2017 14:32:50 +1100 Subject: [PATCH 61/92] py: Make FOR_ITER opcode pop 1+4 slots from the stack when finished. The extra 4 slots correspond to the iterator object stored on the stack. --- py/compile.c | 12 +++++++++--- py/emitbc.c | 7 +------ py/vm.c | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/py/compile.c b/py/compile.c index d2fa03f01e..4fde278d0f 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2887,7 +2887,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn EMIT(yield_value); EMIT(pop_top); } else { - EMIT_ARG(store_comp, comp->scope_cur->kind, for_depth + 2); + EMIT_ARG(store_comp, comp->scope_cur->kind, 5 * for_depth + 6); } } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn_iter, PN_comp_if)) { // if condition @@ -2900,13 +2900,13 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn // for loop mp_parse_node_struct_t *pns_comp_for2 = (mp_parse_node_struct_t*)pn_iter; compile_node(comp, pns_comp_for2->nodes[1]); - EMIT_ARG(get_iter, false); + EMIT_ARG(get_iter, true); compile_scope_comp_iter(comp, pns_comp_for2, pn_inner_expr, for_depth + 1); } EMIT_ARG(jump, l_top); EMIT_ARG(label_assign, l_end); - EMIT_ARG(for_iter_end, false); + EMIT_ARG(for_iter_end, true); } STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { @@ -3070,6 +3070,12 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { #endif } + // dummy 4 objects + EMIT(load_null); + EMIT(load_null); + EMIT(load_null); + EMIT(load_null); + compile_load_id(comp, qstr_arg); compile_scope_comp_iter(comp, pns_comp_for, pns->nodes[0], 0); diff --git a/py/emitbc.c b/py/emitbc.c index 3e0c0b3972..0d7e6519b3 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -788,12 +788,7 @@ void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) { } void mp_emit_bc_for_iter_end(emit_t *emit, bool use_stack) { - emit_bc_pre(emit, -1); - if (use_stack) { - for (size_t i = 0; i < sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); ++i) { - mp_emit_bc_pop_top(emit); - } - } + emit_bc_pre(emit, use_stack ? -1 - sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : -1); } void mp_emit_bc_pop_block(emit_t *emit) { diff --git a/py/vm.c b/py/vm.c index 848a77a453..f9eb2692dc 100644 --- a/py/vm.c +++ b/py/vm.c @@ -744,7 +744,7 @@ unwind_jump:; assert(TOP()); mp_obj_t value = mp_iternext_allow_raise(TOP()); if (value == MP_OBJ_STOP_ITERATION) { - --sp; // pop the exhausted iterator + sp -= 5; // pop the exhausted iterator ip += ulab; // jump to after for-block } else { PUSH(value); // push the next iteration value @@ -1294,7 +1294,7 @@ exception_handler: const byte *ip = code_state->ip + 1; DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward code_state->ip = ip + ulab; // jump to after for-block - code_state->sp -= 1; // pop the exhausted iterator + code_state->sp -= 5; // pop the exhausted iterator goto outer_dispatch_loop; // continue with dispatch loop } else if (*code_state->ip == MP_BC_YIELD_FROM) { // StopIteration inside yield from call means return a value of From 088740ecc40476fd0796a3ef6a68ee7c677eae64 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 17 Jan 2017 15:27:37 +1100 Subject: [PATCH 62/92] py: Optimise storage of iterator so it takes only 4 slots on Py stack. --- py/compile.c | 8 ++++---- py/emitbc.c | 6 +++--- py/emitnative.c | 21 +++++++++------------ py/nativeglue.c | 30 ++++++++++++++++++++++++++++-- py/runtime0.h | 4 ++-- py/vm.c | 25 +++++++++++++++++++------ 6 files changed, 65 insertions(+), 29 deletions(-) diff --git a/py/compile.c b/py/compile.c index 4fde278d0f..5ea7bb4b29 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2887,7 +2887,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn EMIT(yield_value); EMIT(pop_top); } else { - EMIT_ARG(store_comp, comp->scope_cur->kind, 5 * for_depth + 6); + EMIT_ARG(store_comp, comp->scope_cur->kind, 4 * for_depth + 5); } } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pn_iter, PN_comp_if)) { // if condition @@ -3070,13 +3070,13 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { #endif } - // dummy 4 objects - EMIT(load_null); + // There are 4 slots on the stack for the iterator, and the first one is + // NULL to indicate that the second one points to the iterator object. EMIT(load_null); + compile_load_id(comp, qstr_arg); EMIT(load_null); EMIT(load_null); - compile_load_id(comp, qstr_arg); compile_scope_comp_iter(comp, pns_comp_for, pns->nodes[0], 0); if (scope->kind == SCOPE_GEN_EXPR) { diff --git a/py/emitbc.c b/py/emitbc.c index 0d7e6519b3..caa6761fe7 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -735,7 +735,7 @@ void mp_emit_bc_unwind_jump(emit_t *emit, mp_uint_t label, mp_uint_t except_dept // need to pop the iterator if we are breaking out of a for loop emit_write_bytecode_byte(emit, MP_BC_POP_TOP); // also pop the iter_buf - for (size_t i = 0; i < sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); ++i) { + for (size_t i = 0; i < sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) - 1; ++i) { emit_write_bytecode_byte(emit, MP_BC_POP_TOP); } } @@ -778,7 +778,7 @@ void mp_emit_bc_end_finally(emit_t *emit) { } void mp_emit_bc_get_iter(emit_t *emit, bool use_stack) { - emit_bc_pre(emit, use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : 0); + emit_bc_pre(emit, use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) - 1 : 0); emit_write_bytecode_byte(emit, use_stack ? MP_BC_GET_ITER_STACK : MP_BC_GET_ITER); } @@ -788,7 +788,7 @@ void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) { } void mp_emit_bc_for_iter_end(emit_t *emit, bool use_stack) { - emit_bc_pre(emit, use_stack ? -1 - sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : -1); + emit_bc_pre(emit, -(use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : 1)); } void mp_emit_bc_pop_block(emit_t *emit) { diff --git a/py/emitnative.c b/py/emitnative.c index 6e8a37bdea..ffc4d91870 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -105,8 +105,8 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3, [MP_F_CALL_METHOD_N_KW] = 3, [MP_F_CALL_METHOD_N_KW_VAR] = 3, - [MP_F_GETITER] = 1, - [MP_F_ITERNEXT] = 1, + [MP_F_NATIVE_GETITER] = 2, + [MP_F_NATIVE_ITERNEXT] = 1, [MP_F_NLR_PUSH] = 1, [MP_F_NLR_POP] = 0, [MP_F_NATIVE_RAISE] = 1, @@ -1808,20 +1808,20 @@ STATIC void emit_native_get_iter(emit_t *emit, bool use_stack) { assert(vtype == VTYPE_PYOBJ); if (use_stack) { emit_get_stack_pointer_to_reg_for_push(emit, REG_ARG_2, sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t)); + emit_call(emit, MP_F_NATIVE_GETITER); } else { // mp_getiter will allocate the iter_buf on the heap ASM_MOV_IMM_TO_REG(emit->as, 0, REG_ARG_2); + emit_call(emit, MP_F_NATIVE_GETITER); + emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } - emit_call(emit, MP_F_GETITER); - emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) { emit_native_pre(emit); - vtype_kind_t vtype; - emit_access_stack(emit, 1, &vtype, REG_ARG_1); - assert(vtype == VTYPE_PYOBJ); - emit_call(emit, MP_F_ITERNEXT); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_1, sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t)); + adjust_stack(emit, 4); + emit_call(emit, MP_F_NATIVE_ITERNEXT); ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)MP_OBJ_STOP_ITERATION, REG_TEMP1); ASM_JUMP_IF_REG_EQ(emit->as, REG_RET, REG_TEMP1, label); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); @@ -1830,10 +1830,7 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) { STATIC void emit_native_for_iter_end(emit_t *emit, bool use_stack) { // adjust stack counter (we get here from for_iter ending, which popped the value for us) emit_native_pre(emit); - adjust_stack(emit, -1); - if (use_stack) { - adjust_stack(emit, -(sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t))); - } + adjust_stack(emit, -(use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : 1)); emit_post(emit); } diff --git a/py/nativeglue.c b/py/nativeglue.c index 5db63080b4..694dfca74a 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -98,6 +98,32 @@ void mp_native_raise(mp_obj_t o) { } } +// wrapper that handles iterator buffer +STATIC mp_obj_t mp_native_getiter(mp_obj_t obj, mp_obj_iter_buf_t *iter) { + if (iter == NULL) { + return mp_getiter(obj, NULL); + } else { + obj = mp_getiter(obj, iter); + if (obj != MP_OBJ_FROM_PTR(iter)) { + // Iterator didn't use the stack so indicate that with MP_OBJ_NULL. + iter->base.type = MP_OBJ_NULL; + iter->buf[0] = obj; + } + return NULL; + } +} + +// wrapper that handles iterator buffer +STATIC mp_obj_t mp_native_iternext(mp_obj_iter_buf_t *iter) { + mp_obj_t obj; + if (iter->base.type == MP_OBJ_NULL) { + obj = iter->buf[0]; + } else { + obj = MP_OBJ_FROM_PTR(iter); + } + return mp_iternext(obj); +} + // these must correspond to the respective enum in runtime0.h void *const mp_fun_table[MP_F_NUMBER_OF] = { mp_convert_obj_to_native, @@ -127,8 +153,8 @@ void *const mp_fun_table[MP_F_NUMBER_OF] = { mp_native_call_function_n_kw, mp_call_method_n_kw, mp_call_method_n_kw_var, - mp_getiter, - mp_iternext, + mp_native_getiter, + mp_native_iternext, nlr_push, nlr_pop, mp_native_raise, diff --git a/py/runtime0.h b/py/runtime0.h index 8d62403a7c..b1ed710262 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -127,8 +127,8 @@ typedef enum { MP_F_NATIVE_CALL_FUNCTION_N_KW, MP_F_CALL_METHOD_N_KW, MP_F_CALL_METHOD_N_KW_VAR, - MP_F_GETITER, - MP_F_ITERNEXT, + MP_F_NATIVE_GETITER, + MP_F_NATIVE_ITERNEXT, MP_F_NLR_PUSH, MP_F_NLR_POP, MP_F_NATIVE_RAISE, diff --git a/py/vm.c b/py/vm.c index f9eb2692dc..7a906cd804 100644 --- a/py/vm.c +++ b/py/vm.c @@ -728,12 +728,20 @@ unwind_jump:; SET_TOP(mp_getiter(TOP(), NULL)); DISPATCH(); + // An iterator for a for-loop takes 4 slots on the stack. They are either + // used to store the iterator object itself, or the first slot is NULL and + // the second slot holds a reference to the iterator object. ENTRY(MP_BC_GET_ITER_STACK): { MARK_EXC_IP_SELECTIVE(); mp_obj_t obj = TOP(); mp_obj_iter_buf_t *iter_buf = (mp_obj_iter_buf_t*)sp; - sp += sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t); - SET_TOP(mp_getiter(obj, iter_buf)); + sp += sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) - 1; + obj = mp_getiter(obj, iter_buf); + if (obj != MP_OBJ_FROM_PTR(iter_buf)) { + // Iterator didn't use the stack so indicate that with MP_OBJ_NULL. + sp[-3] = MP_OBJ_NULL; + sp[-2] = obj; + } DISPATCH(); } @@ -741,10 +749,15 @@ unwind_jump:; MARK_EXC_IP_SELECTIVE(); DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward code_state->sp = sp; - assert(TOP()); - mp_obj_t value = mp_iternext_allow_raise(TOP()); + mp_obj_t obj; + if (sp[-3] == MP_OBJ_NULL) { + obj = sp[-2]; + } else { + obj = MP_OBJ_FROM_PTR(&sp[-3]); + } + mp_obj_t value = mp_iternext_allow_raise(obj); if (value == MP_OBJ_STOP_ITERATION) { - sp -= 5; // pop the exhausted iterator + sp -= 4; // pop the exhausted iterator ip += ulab; // jump to after for-block } else { PUSH(value); // push the next iteration value @@ -1294,7 +1307,7 @@ exception_handler: const byte *ip = code_state->ip + 1; DECODE_ULABEL; // the jump offset if iteration finishes; for labels are always forward code_state->ip = ip + ulab; // jump to after for-block - code_state->sp -= 5; // pop the exhausted iterator + code_state->sp -= 4; // pop the exhausted iterator goto outer_dispatch_loop; // continue with dispatch loop } else if (*code_state->ip == MP_BC_YIELD_FROM) { // StopIteration inside yield from call means return a value of From 30b42dd72d4aa310e8c96ccbaf75150f8649c262 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 17 Jan 2017 15:30:18 +1100 Subject: [PATCH 63/92] py: Remove unused "use_stack" argument from for_iter_end emit function. --- py/compile.c | 4 ++-- py/emit.h | 4 ++-- py/emitbc.c | 4 ++-- py/emitnative.c | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/py/compile.c b/py/compile.c index 5ea7bb4b29..3967350e38 100644 --- a/py/compile.c +++ b/py/compile.c @@ -1484,7 +1484,7 @@ STATIC void compile_for_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { EMIT_ARG(jump, continue_label); } EMIT_ARG(label_assign, pop_label); - EMIT_ARG(for_iter_end, true); + EMIT(for_iter_end); // break/continue apply to outer loop (if any) in the else block END_BREAK_CONTINUE_BLOCK @@ -2906,7 +2906,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn EMIT_ARG(jump, l_top); EMIT_ARG(label_assign, l_end); - EMIT_ARG(for_iter_end, true); + EMIT(for_iter_end); } STATIC void check_for_doc_string(compiler_t *comp, mp_parse_node_t pn) { diff --git a/py/emit.h b/py/emit.h index 6acae9e80f..64bb957f60 100644 --- a/py/emit.h +++ b/py/emit.h @@ -112,7 +112,7 @@ typedef struct _emit_method_table_t { void (*end_finally)(emit_t *emit); void (*get_iter)(emit_t *emit, bool use_stack); void (*for_iter)(emit_t *emit, mp_uint_t label); - void (*for_iter_end)(emit_t *emit, bool use_stack); + void (*for_iter_end)(emit_t *emit); void (*pop_block)(emit_t *emit); void (*pop_except)(emit_t *emit); void (*unary_op)(emit_t *emit, mp_unary_op_t op); @@ -230,7 +230,7 @@ void mp_emit_bc_setup_finally(emit_t *emit, mp_uint_t label); void mp_emit_bc_end_finally(emit_t *emit); void mp_emit_bc_get_iter(emit_t *emit, bool use_stack); void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label); -void mp_emit_bc_for_iter_end(emit_t *emit, bool use_stack); +void mp_emit_bc_for_iter_end(emit_t *emit); void mp_emit_bc_pop_block(emit_t *emit); void mp_emit_bc_pop_except(emit_t *emit); void mp_emit_bc_unary_op(emit_t *emit, mp_unary_op_t op); diff --git a/py/emitbc.c b/py/emitbc.c index caa6761fe7..0076c147e1 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -787,8 +787,8 @@ void mp_emit_bc_for_iter(emit_t *emit, mp_uint_t label) { emit_write_bytecode_byte_unsigned_label(emit, MP_BC_FOR_ITER, label); } -void mp_emit_bc_for_iter_end(emit_t *emit, bool use_stack) { - emit_bc_pre(emit, -(use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : 1)); +void mp_emit_bc_for_iter_end(emit_t *emit) { + emit_bc_pre(emit, -(sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t))); } void mp_emit_bc_pop_block(emit_t *emit) { diff --git a/py/emitnative.c b/py/emitnative.c index ffc4d91870..9176f0a3c2 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1827,10 +1827,10 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) { emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } -STATIC void emit_native_for_iter_end(emit_t *emit, bool use_stack) { +STATIC void emit_native_for_iter_end(emit_t *emit) { // adjust stack counter (we get here from for_iter ending, which popped the value for us) emit_native_pre(emit); - adjust_stack(emit, -(use_stack ? sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t) : 1)); + adjust_stack(emit, -(sizeof(mp_obj_iter_buf_t) / sizeof(mp_obj_t))); emit_post(emit); } From cb6300697c7ea7f3634375f348982fa49a6df24f Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 17 Jan 2017 15:38:32 +1100 Subject: [PATCH 64/92] py/runtime: Optimise case of identity iterator so it doesn't alloc RAM. --- py/runtime.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/py/runtime.c b/py/runtime.c index b9d7b72dcf..b792eabe7c 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1103,6 +1103,13 @@ void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t value) { mp_obj_t mp_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { assert(o_in); + mp_obj_type_t *type = mp_obj_get_type(o_in); + + // Check for native getiter which is the identity. We handle this case explicitly + // so we don't unnecessarily allocate any RAM for the iter_buf, which won't be used. + if (type->getiter == mp_identity_getiter) { + return o_in; + } // if caller did not provide a buffer then allocate one on the heap if (iter_buf == NULL) { @@ -1110,7 +1117,6 @@ mp_obj_t mp_getiter(mp_obj_t o_in, mp_obj_iter_buf_t *iter_buf) { } // check for native getiter (corresponds to __iter__) - mp_obj_type_t *type = mp_obj_get_type(o_in); if (type->getiter != NULL) { mp_obj_t iter = type->getiter(o_in, iter_buf); if (iter != MP_OBJ_NULL) { From 861b00178370d507516273ea95dd48acd31f36ff Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 18 Jan 2017 16:53:15 +1100 Subject: [PATCH 65/92] tests/cmdline: Update tests to pass with latest changes to bytecode. --- tests/cmdline/cmd_parsetree.py.exp | 4 +- tests/cmdline/cmd_showbc.py.exp | 81 ++++++++++++++++-------------- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index 06fbeadfc8..5510d11cd0 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -31,7 +31,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## \.\+5b arg names: -(N_STATE 2) +(N_STATE 5) (N_EXC_STACK 0) bc=-1 line=1 bc=0 line=4 @@ -43,7 +43,7 @@ arg names: bc=32 line=10 bc=37 line=11 00 BUILD_TUPLE 0 -02 GET_ITER +02 GET_ITER_STACK 03 FOR_ITER 12 06 STORE_NAME i 09 JUMP 3 diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 8ee47eab66..68e2fe51a1 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -31,7 +31,7 @@ File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ byt Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## \.\+rg names: -(N_STATE 23) +(N_STATE 22) (N_EXC_STACK 2) (INIT_CELL 14) (INIT_CELL 15) @@ -251,10 +251,6 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): \\d\+ LOAD_FAST 1 \\d\+ POP_TOP \\d\+ JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ POP_TOP -\\d\+ POP_TOP -\\d\+ POP_TOP \\d\+ SETUP_FINALLY \\d\+ \\d\+ SETUP_EXCEPT \\d\+ \\d\+ JUMP \\d\+ @@ -439,58 +435,67 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## \.\+5b arg names: * * * -(N_STATE 6) +(N_STATE 9) (N_EXC_STACK 0) bc=-\\d\+ line=1 -00 LOAD_FAST 2 -01 FOR_ITER 17 -04 STORE_FAST 3 -05 LOAD_DEREF 1 -07 POP_JUMP_IF_FALSE 1 -10 LOAD_DEREF 0 -12 YIELD_VALUE -13 POP_TOP -14 JUMP 1 -17 LOAD_CONST_NONE -18 RETURN_VALUE +00 LOAD_NULL +01 LOAD_FAST 2 +02 LOAD_NULL +03 LOAD_NULL +04 FOR_ITER 20 +07 STORE_FAST 3 +08 LOAD_DEREF 1 +10 POP_JUMP_IF_FALSE 4 +13 LOAD_DEREF 0 +15 YIELD_VALUE +16 POP_TOP +17 JUMP 4 +20 LOAD_CONST_NONE +21 RETURN_VALUE File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## \.\+5b arg names: * * * -(N_STATE 7) +(N_STATE 10) (N_EXC_STACK 0) bc=-\\d\+ line=1 00 BUILD_LIST 0 -02 LOAD_FAST 2 -03 FOR_ITER 19 -06 STORE_FAST 3 -07 LOAD_DEREF 1 -09 POP_JUMP_IF_FALSE 3 -12 LOAD_DEREF 0 -14 STORE_COMP 8 -16 JUMP 3 -19 RETURN_VALUE +02 LOAD_NULL +03 LOAD_FAST 2 +04 LOAD_NULL +05 LOAD_NULL +06 FOR_ITER 22 +09 STORE_FAST 3 +10 LOAD_DEREF 1 +12 POP_JUMP_IF_FALSE 6 +15 LOAD_DEREF 0 +17 STORE_COMP 20 +19 JUMP 6 +22 RETURN_VALUE File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## \.\+5b arg names: * * * -(N_STATE 8) +(N_STATE 11) (N_EXC_STACK 0) bc=-\\d\+ line=1 ######## 00 BUILD_MAP 0 -02 LOAD_FAST 2 -03 FOR_ITER 21 -06 STORE_FAST 3 -07 LOAD_DEREF 1 -09 POP_JUMP_IF_FALSE 3 -12 LOAD_DEREF 0 -14 LOAD_DEREF 0 -16 STORE_COMP 13 -18 JUMP 3 -21 RETURN_VALUE +02 LOAD_NULL +03 LOAD_FAST 2 +04 LOAD_NULL +05 LOAD_NULL +06 FOR_ITER 24 +09 STORE_FAST 3 +10 LOAD_DEREF 1 +12 POP_JUMP_IF_FALSE 6 +15 LOAD_DEREF 0 +17 LOAD_DEREF 0 +19 STORE_COMP 25 +21 JUMP 6 +24 RETURN_VALUE File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## From 4d2bab14445b7261e4598f2aad5032d08e95b3ef Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 10 Feb 2017 15:39:33 +1100 Subject: [PATCH 66/92] py/compile: Optimise list/dict/set comprehensions to use stack iter. --- py/compile.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/py/compile.c b/py/compile.c index 3967350e38..58f6631c4b 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2372,7 +2372,9 @@ STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, close_over_variables_etc(comp, this_scope, 0, 0); compile_node(comp, pns_comp_for->nodes[1]); // source of the iterator - EMIT_ARG(get_iter, false); + if (kind == SCOPE_GEN_EXPR) { + EMIT_ARG(get_iter, false); + } EMIT_ARG(call_function, 1, 0, 0); } @@ -3072,10 +3074,15 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { // There are 4 slots on the stack for the iterator, and the first one is // NULL to indicate that the second one points to the iterator object. - EMIT(load_null); - compile_load_id(comp, qstr_arg); - EMIT(load_null); - EMIT(load_null); + if (scope->kind == SCOPE_GEN_EXPR) { + EMIT(load_null); + compile_load_id(comp, qstr_arg); + EMIT(load_null); + EMIT(load_null); + } else { + compile_load_id(comp, qstr_arg); + EMIT_ARG(get_iter, true); + } compile_scope_comp_iter(comp, pns_comp_for, pns->nodes[0], 0); From 86b3db9cd08c6ec647fae7a1c723e902f47b7e1e Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 10 Feb 2017 15:39:55 +1100 Subject: [PATCH 67/92] tests/cmdline/cmd_showbc: Update to work with recent changes. --- tests/cmdline/cmd_showbc.py.exp | 46 ++++++++++++++------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 68e2fe51a1..d0baee10f8 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -175,14 +175,12 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): \\d\+ LOAD_FAST 15 \\d\+ MAKE_CLOSURE \.\+ 2 \\d\+ LOAD_FAST 2 -\\d\+ GET_ITER \\d\+ CALL_FUNCTION n=1 nkw=0 \\d\+ STORE_FAST 0 \\d\+ LOAD_FAST 14 \\d\+ LOAD_FAST 15 \\d\+ MAKE_CLOSURE \.\+ 2 \\d\+ LOAD_FAST 2 -\\d\+ GET_ITER \\d\+ CALL_FUNCTION n=1 nkw=0 \\d\+ STORE_FAST 0 \\d\+ LOAD_FAST 0 @@ -461,18 +459,16 @@ arg names: * * * (N_EXC_STACK 0) bc=-\\d\+ line=1 00 BUILD_LIST 0 -02 LOAD_NULL -03 LOAD_FAST 2 -04 LOAD_NULL -05 LOAD_NULL -06 FOR_ITER 22 -09 STORE_FAST 3 -10 LOAD_DEREF 1 -12 POP_JUMP_IF_FALSE 6 -15 LOAD_DEREF 0 -17 STORE_COMP 20 -19 JUMP 6 -22 RETURN_VALUE +02 LOAD_FAST 2 +03 GET_ITER_STACK +04 FOR_ITER 20 +07 STORE_FAST 3 +08 LOAD_DEREF 1 +10 POP_JUMP_IF_FALSE 4 +13 LOAD_DEREF 0 +15 STORE_COMP 20 +17 JUMP 4 +20 RETURN_VALUE File cmdline/cmd_showbc.py, code block '' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## @@ -483,19 +479,17 @@ arg names: * * * bc=-\\d\+ line=1 ######## 00 BUILD_MAP 0 -02 LOAD_NULL -03 LOAD_FAST 2 -04 LOAD_NULL -05 LOAD_NULL -06 FOR_ITER 24 -09 STORE_FAST 3 -10 LOAD_DEREF 1 -12 POP_JUMP_IF_FALSE 6 +02 LOAD_FAST 2 +03 GET_ITER_STACK +04 FOR_ITER 22 +07 STORE_FAST 3 +08 LOAD_DEREF 1 +10 POP_JUMP_IF_FALSE 4 +13 LOAD_DEREF 0 15 LOAD_DEREF 0 -17 LOAD_DEREF 0 -19 STORE_COMP 25 -21 JUMP 6 -24 RETURN_VALUE +17 STORE_COMP 25 +19 JUMP 4 +22 RETURN_VALUE File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ bytes) Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): ######## From 019048a6dc0783515e7fba4255dd2b22265d1059 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 10 Feb 2017 16:09:20 +1100 Subject: [PATCH 68/92] tests/micropython: Add test for iterating with the heap locked. --- tests/micropython/heapalloc_iter.py | 49 +++++++++++++++++++++++++++++ tests/run-tests | 1 + 2 files changed, 50 insertions(+) create mode 100644 tests/micropython/heapalloc_iter.py diff --git a/tests/micropython/heapalloc_iter.py b/tests/micropython/heapalloc_iter.py new file mode 100644 index 0000000000..3315d78c38 --- /dev/null +++ b/tests/micropython/heapalloc_iter.py @@ -0,0 +1,49 @@ +# test that iterating doesn't use the heap + +try: + from micropython import heap_lock, heap_unlock +except (ImportError, AttributeError): + heap_lock = heap_unlock = lambda:0 +import array + +def do_iter(l): + for i in l: + print(i) + +def gen_func(): + yield 1 + yield 2 + +# pre-create collections to iterate over +ba = bytearray(b'123') +ar = array.array('H', (123, 456)) +t = (1, 2, 3) +l = [1, 2] +d = {1:2} +s = {1} +fs = frozenset((1,)) +g1 = (100 + x for x in range(2)) +g2 = gen_func() + +# test certain builtins with the heap locked +heap_lock() +print(all(t)) +print(any(t)) +print(min(t)) +print(max(t)) +print(sum(t)) +heap_unlock() + +# test iterating over collections with the heap locked +heap_lock() +do_iter(b'123') +do_iter(ba) +do_iter(ar) +do_iter(t) +do_iter(l) +do_iter(d) +do_iter(s) +do_iter(fs) +do_iter(g1) +do_iter(g2) +heap_unlock() diff --git a/tests/run-tests b/tests/run-tests index 4bebe8d47f..6cca829b67 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -312,6 +312,7 @@ def run_tests(pyb, tests, args): skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info + skip_tests.add('micropython/heapalloc_iter.py') # requires generators for test_file in tests: test_file = test_file.replace('\\', '/') From e6003f466e6f5b7eddd01740666230b62ab5a872 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 13 Feb 2017 15:44:31 +1100 Subject: [PATCH 69/92] py: De-optimise some uses of mp_getiter, so they don't use the C stack. In these cases the heap is anyway used to create a new object so no real need to use the C stack for iterating. It saves a few bytes of code size. --- py/objarray.c | 3 +-- py/objdict.c | 9 +++------ py/objlist.c | 3 +-- py/objset.c | 15 +++++---------- py/objtuple.c | 3 +-- py/runtime.c | 6 ++---- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/py/objarray.c b/py/objarray.c index c81aebb50c..1b590f3c05 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -141,8 +141,7 @@ STATIC mp_obj_t array_construct(char typecode, mp_obj_t initializer) { mp_obj_array_t *array = array_new(typecode, len); - mp_obj_iter_buf_t iter_buf; - mp_obj_t iterable = mp_getiter(initializer, &iter_buf); + mp_obj_t iterable = mp_getiter(initializer, NULL); mp_obj_t item; size_t i = 0; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { diff --git a/py/objdict.c b/py/objdict.c index c52403f715..013cc0a045 100644 --- a/py/objdict.c +++ b/py/objdict.c @@ -250,8 +250,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(dict_copy_obj, dict_copy); // this is a classmethod STATIC mp_obj_t dict_fromkeys(size_t n_args, const mp_obj_t *args) { - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(args[1], &iter_buf); + mp_obj_t iter = mp_getiter(args[1], NULL); mp_obj_t value = mp_const_none; mp_obj_t next = MP_OBJ_NULL; @@ -377,12 +376,10 @@ STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwarg } } else { // update from a generic iterable of pairs - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(args[1], &iter_buf); + mp_obj_t iter = mp_getiter(args[1], NULL); mp_obj_t next = MP_OBJ_NULL; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { - mp_obj_iter_buf_t inner_iter_buf; - mp_obj_t inneriter = mp_getiter(next, &inner_iter_buf); + mp_obj_t inneriter = mp_getiter(next, NULL); mp_obj_t key = mp_iternext(inneriter); mp_obj_t value = mp_iternext(inneriter); mp_obj_t stop = mp_iternext(inneriter); diff --git a/py/objlist.c b/py/objlist.c index 28da109912..55ee191207 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -60,8 +60,7 @@ STATIC void list_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t k } STATIC mp_obj_t list_extend_from_iter(mp_obj_t list, mp_obj_t iterable) { - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(iterable, &iter_buf); + mp_obj_t iter = mp_getiter(iterable, NULL); mp_obj_t item; while ((item = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { mp_obj_list_append(list, item); diff --git a/py/objset.c b/py/objset.c index 99e1e8ca57..f74bc74a07 100644 --- a/py/objset.c +++ b/py/objset.c @@ -129,8 +129,7 @@ STATIC mp_obj_t set_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ default: { // can only be 0 or 1 arg // 1 argument, an iterable from which we make a new set mp_obj_t set = mp_obj_new_set(0, NULL); - mp_obj_iter_buf_t iter_buf; - mp_obj_t iterable = mp_getiter(args[0], &iter_buf); + mp_obj_t iterable = mp_getiter(args[0], NULL); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { mp_obj_set_store(set, item); @@ -235,8 +234,7 @@ STATIC mp_obj_t set_diff_int(size_t n_args, const mp_obj_t *args, bool update) { if (self == other) { set_clear(self); } else { - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(other, &iter_buf); + mp_obj_t iter = mp_getiter(other, NULL); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { set_discard(self, next); @@ -273,8 +271,7 @@ STATIC mp_obj_t set_intersect_int(mp_obj_t self_in, mp_obj_t other, bool update) mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); mp_obj_set_t *out = MP_OBJ_TO_PTR(mp_obj_new_set(0, NULL)); - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(other, &iter_buf); + mp_obj_t iter = mp_getiter(other, NULL); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { if (mp_set_lookup(&self->set, next, MP_MAP_LOOKUP)) { @@ -414,8 +411,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(set_remove_obj, set_remove); STATIC mp_obj_t set_symmetric_difference_update(mp_obj_t self_in, mp_obj_t other_in) { check_set(self_in); mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(other_in, &iter_buf); + mp_obj_t iter = mp_getiter(other_in, NULL); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { mp_set_lookup(&self->set, next, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND_OR_REMOVE_IF_FOUND); @@ -434,8 +430,7 @@ STATIC mp_obj_t set_symmetric_difference(mp_obj_t self_in, mp_obj_t other_in) { STATIC MP_DEFINE_CONST_FUN_OBJ_2(set_symmetric_difference_obj, set_symmetric_difference); STATIC void set_update_int(mp_obj_set_t *self, mp_obj_t other_in) { - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(other_in, &iter_buf); + mp_obj_t iter = mp_getiter(other_in, NULL); mp_obj_t next; while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { mp_set_lookup(&self->set, next, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); diff --git a/py/objtuple.c b/py/objtuple.c index ad45696cae..b28807c0d0 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -82,8 +82,7 @@ STATIC mp_obj_t mp_obj_tuple_make_new(const mp_obj_type_t *type_in, size_t n_arg size_t len = 0; mp_obj_t *items = m_new(mp_obj_t, alloc); - mp_obj_iter_buf_t iter_buf; - mp_obj_t iterable = mp_getiter(args[0], &iter_buf); + mp_obj_t iterable = mp_getiter(args[0], NULL); mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (len >= alloc) { diff --git a/py/runtime.c b/py/runtime.c index b792eabe7c..db6a6f18f9 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -745,8 +745,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // get the keys iterable mp_obj_t dest[3]; mp_load_method(kw_dict, MP_QSTR_keys, dest); - mp_obj_iter_buf_t iter_buf; - mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest), &iter_buf); + mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest), NULL); mp_obj_t key; while ((key = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { @@ -877,8 +876,7 @@ void mp_unpack_ex(mp_obj_t seq_in, size_t num_in, mp_obj_t *items) { // items destination array, then the rest to a dynamically created list. Once the // iterable is exhausted, we take from this list for the right part of the items. // TODO Improve to waste less memory in the dynamically created list. - mp_obj_iter_buf_t iter_buf; - mp_obj_t iterable = mp_getiter(seq_in, &iter_buf); + mp_obj_t iterable = mp_getiter(seq_in, NULL); mp_obj_t item; for (seq_len = 0; seq_len < num_left; seq_len++) { item = mp_iternext(iterable); From 7839b8b827199ce593bfb87bc62b426a2798fde6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 13 Feb 2017 15:45:24 +1100 Subject: [PATCH 70/92] tests/micropython/heapalloc_iter: Add tests for contains and unpack. --- tests/micropython/heapalloc_iter.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/micropython/heapalloc_iter.py b/tests/micropython/heapalloc_iter.py index 3315d78c38..e1ed3daa3b 100644 --- a/tests/micropython/heapalloc_iter.py +++ b/tests/micropython/heapalloc_iter.py @@ -25,6 +25,22 @@ fs = frozenset((1,)) g1 = (100 + x for x in range(2)) g2 = gen_func() +# test containment (both success and failure) with the heap locked +heap_lock() +print(49 in b'123', 255 in b'123') +print(1 in t, -1 in t) +print(1 in l, -1 in l) +print(1 in d, -1 in d) +print(1 in s, -1 in s) +heap_unlock() + +# test unpacking with the heap locked +unp0 = unp1 = unp2 = None # preallocate slots for globals +heap_lock() +unp0, unp1, unp2 = t +print(unp0, unp1, unp2) +heap_unlock() + # test certain builtins with the heap locked heap_lock() print(all(t)) From 71019ae4f5ba8819af27152198afc0274085c8a9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 10:58:05 +1100 Subject: [PATCH 71/92] py/grammar: Group no-compile grammar rules together to shrink tables. Grammar rules have 2 variants: ones that are attached to a specific compile function which is called to compile that grammar node, and ones that don't have a compile function and are instead just inspected to see what form they take. In the compiler there is a table of all grammar rules, with each entry having a pointer to the associated compile function. Those rules with no compile function have a null pointer. There are 120 such rules, so that's 120 words of essentially wasted code space. By grouping together the compile vs no-compile rules we can put all the no-compile rules at the end of the list of rules, and then we don't need to store the null pointers. We just have a truncated table and it's guaranteed that when indexing this table we only index the first half, the half with populated pointers. This patch implements such a grouping by having a specific macro for the compile vs no-compile grammar rules (DEF_RULE vs DEF_RULE_NC). It saves around 460 bytes of code on 32-bit archs. --- py/compile.c | 18 +++- py/emitinlinethumb.c | 13 ++- py/grammar.h | 240 +++++++++++++++++++++---------------------- py/parse.c | 28 ++++- 4 files changed, 171 insertions(+), 128 deletions(-) diff --git a/py/compile.c b/py/compile.c index 58f6631c4b..3fb45cd69c 100644 --- a/py/compile.c +++ b/py/compile.c @@ -41,13 +41,21 @@ // TODO need to mangle __attr names typedef enum { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) PN_##rule, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE - PN_maximum_number_of, +#undef DEF_RULE_NC PN_string, // special node for non-interned string PN_bytes, // special node for non-interned bytes PN_const_object, // special node for a constant, generic Python object +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) PN_##rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC } pn_kind_t; #define NEED_METHOD_TABLE MICROPY_EMIT_NATIVE @@ -2680,14 +2688,14 @@ STATIC void compile_const_object(compiler_t *comp, mp_parse_node_struct_t *pns) typedef void (*compile_function_t)(compiler_t*, mp_parse_node_struct_t*); STATIC const compile_function_t compile_function[] = { -#define nc NULL +// only define rules with a compile function #define c(f) compile_##f #define DEF_RULE(rule, comp, kind, ...) comp, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" -#undef nc #undef c #undef DEF_RULE - NULL, +#undef DEF_RULE_NC compile_string, compile_bytes, compile_const_object, @@ -2743,8 +2751,8 @@ STATIC void compile_node(compiler_t *comp, mp_parse_node_t pn) { } else { mp_parse_node_struct_t *pns = (mp_parse_node_struct_t*)pn; EMIT_ARG(set_source_line, pns->source_line); + assert(MP_PARSE_NODE_STRUCT_KIND(pns) <= PN_const_object); compile_function_t f = compile_function[MP_PARSE_NODE_STRUCT_KIND(pns)]; - assert(f != NULL); f(comp, pns); } } diff --git a/py/emitinlinethumb.c b/py/emitinlinethumb.c index 4062728136..89bcebfead 100644 --- a/py/emitinlinethumb.c +++ b/py/emitinlinethumb.c @@ -36,10 +36,21 @@ #if MICROPY_EMIT_INLINE_THUMB typedef enum { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) PN_##rule, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE - PN_maximum_number_of, +#undef DEF_RULE_NC + PN_string, // special node for non-interned string + PN_bytes, // special node for non-interned bytes + PN_const_object, // special node for a constant, generic Python object +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) PN_##rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC } pn_kind_t; struct _emit_inline_asm_t { diff --git a/py/grammar.h b/py/grammar.h index e261aad226..98f6d16aee 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -37,12 +37,12 @@ // file_input: (NEWLINE | stmt)* ENDMARKER // eval_input: testlist NEWLINE* ENDMARKER -DEF_RULE(single_input, nc, or(3), tok(NEWLINE), rule(simple_stmt), rule(compound_stmt)) +DEF_RULE_NC(single_input, or(3), tok(NEWLINE), rule(simple_stmt), rule(compound_stmt)) DEF_RULE(file_input, c(generic_all_nodes), and_ident(1), opt_rule(file_input_2)) DEF_RULE(file_input_2, c(generic_all_nodes), one_or_more, rule(file_input_3)) -DEF_RULE(file_input_3, nc, or(2), tok(NEWLINE), rule(stmt)) -DEF_RULE(eval_input, nc, and_ident(2), rule(testlist), opt_rule(eval_input_2)) -DEF_RULE(eval_input_2, nc, and(1), tok(NEWLINE)) +DEF_RULE_NC(file_input_3, or(2), tok(NEWLINE), rule(stmt)) +DEF_RULE_NC(eval_input, and_ident(2), rule(testlist), opt_rule(eval_input_2)) +DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE)) // decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE // decorators: decorator+ @@ -55,42 +55,42 @@ DEF_RULE(eval_input_2, nc, and(1), tok(NEWLINE)) // varargslist: vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef // vfpdef: NAME -DEF_RULE(decorator, nc, and(4), tok(DEL_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE)) -DEF_RULE(decorators, nc, one_or_more, rule(decorator)) +DEF_RULE_NC(decorator, and(4), tok(DEL_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE)) +DEF_RULE_NC(decorators, one_or_more, rule(decorator)) DEF_RULE(decorated, c(decorated), and_ident(2), rule(decorators), rule(decorated_body)) #if MICROPY_PY_ASYNC_AWAIT -DEF_RULE(decorated_body, nc, or(3), rule(classdef), rule(funcdef), rule(async_funcdef)) -DEF_RULE(async_funcdef, nc, and(2), tok(KW_ASYNC), rule(funcdef)) +DEF_RULE_NC(decorated_body, or(3), rule(classdef), rule(funcdef), rule(async_funcdef)) +DEF_RULE_NC(async_funcdef, and(2), tok(KW_ASYNC), rule(funcdef)) #else -DEF_RULE(decorated_body, nc, or(2), rule(classdef), rule(funcdef)) +DEF_RULE_NC(decorated_body, or(2), rule(classdef), rule(funcdef)) #endif DEF_RULE(funcdef, c(funcdef), and_blank(8), tok(KW_DEF), tok(NAME), tok(DEL_PAREN_OPEN), opt_rule(typedargslist), tok(DEL_PAREN_CLOSE), opt_rule(funcdefrettype), tok(DEL_COLON), rule(suite)) -DEF_RULE(funcdefrettype, nc, and_ident(2), tok(DEL_MINUS_MORE), rule(test)) +DEF_RULE_NC(funcdefrettype, and_ident(2), tok(DEL_MINUS_MORE), rule(test)) // note: typedargslist lets through more than is allowed, compiler does further checks -DEF_RULE(typedargslist, nc, list_with_end, rule(typedargslist_item), tok(DEL_COMMA)) -DEF_RULE(typedargslist_item, nc, or(3), rule(typedargslist_name), rule(typedargslist_star), rule(typedargslist_dbl_star)) -DEF_RULE(typedargslist_name, nc, and_ident(3), tok(NAME), opt_rule(typedargslist_colon), opt_rule(typedargslist_equal)) -DEF_RULE(typedargslist_star, nc, and(2), tok(OP_STAR), opt_rule(tfpdef)) -DEF_RULE(typedargslist_dbl_star, nc, and(3), tok(OP_DBL_STAR), tok(NAME), opt_rule(typedargslist_colon)) -DEF_RULE(typedargslist_colon, nc, and_ident(2), tok(DEL_COLON), rule(test)) -DEF_RULE(typedargslist_equal, nc, and_ident(2), tok(DEL_EQUAL), rule(test)) -DEF_RULE(tfpdef, nc, and(2), tok(NAME), opt_rule(typedargslist_colon)) +DEF_RULE_NC(typedargslist, list_with_end, rule(typedargslist_item), tok(DEL_COMMA)) +DEF_RULE_NC(typedargslist_item, or(3), rule(typedargslist_name), rule(typedargslist_star), rule(typedargslist_dbl_star)) +DEF_RULE_NC(typedargslist_name, and_ident(3), tok(NAME), opt_rule(typedargslist_colon), opt_rule(typedargslist_equal)) +DEF_RULE_NC(typedargslist_star, and(2), tok(OP_STAR), opt_rule(tfpdef)) +DEF_RULE_NC(typedargslist_dbl_star, and(3), tok(OP_DBL_STAR), tok(NAME), opt_rule(typedargslist_colon)) +DEF_RULE_NC(typedargslist_colon, and_ident(2), tok(DEL_COLON), rule(test)) +DEF_RULE_NC(typedargslist_equal, and_ident(2), tok(DEL_EQUAL), rule(test)) +DEF_RULE_NC(tfpdef, and(2), tok(NAME), opt_rule(typedargslist_colon)) // note: varargslist lets through more than is allowed, compiler does further checks -DEF_RULE(varargslist, nc, list_with_end, rule(varargslist_item), tok(DEL_COMMA)) -DEF_RULE(varargslist_item, nc, or(3), rule(varargslist_name), rule(varargslist_star), rule(varargslist_dbl_star)) -DEF_RULE(varargslist_name, nc, and_ident(2), tok(NAME), opt_rule(varargslist_equal)) -DEF_RULE(varargslist_star, nc, and(2), tok(OP_STAR), opt_rule(vfpdef)) -DEF_RULE(varargslist_dbl_star, nc, and(2), tok(OP_DBL_STAR), tok(NAME)) -DEF_RULE(varargslist_equal, nc, and_ident(2), tok(DEL_EQUAL), rule(test)) -DEF_RULE(vfpdef, nc, and_ident(1), tok(NAME)) +DEF_RULE_NC(varargslist, list_with_end, rule(varargslist_item), tok(DEL_COMMA)) +DEF_RULE_NC(varargslist_item, or(3), rule(varargslist_name), rule(varargslist_star), rule(varargslist_dbl_star)) +DEF_RULE_NC(varargslist_name, and_ident(2), tok(NAME), opt_rule(varargslist_equal)) +DEF_RULE_NC(varargslist_star, and(2), tok(OP_STAR), opt_rule(vfpdef)) +DEF_RULE_NC(varargslist_dbl_star, and(2), tok(OP_DBL_STAR), tok(NAME)) +DEF_RULE_NC(varargslist_equal, and_ident(2), tok(DEL_EQUAL), rule(test)) +DEF_RULE_NC(vfpdef, and_ident(1), tok(NAME)) // stmt: compound_stmt | simple_stmt -DEF_RULE(stmt, nc, or(2), rule(compound_stmt), rule(simple_stmt)) +DEF_RULE_NC(stmt, or(2), rule(compound_stmt), rule(simple_stmt)) // simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -DEF_RULE(simple_stmt, nc, and_ident(2), rule(simple_stmt_2), tok(NEWLINE)) +DEF_RULE_NC(simple_stmt, and_ident(2), rule(simple_stmt_2), tok(NEWLINE)) DEF_RULE(simple_stmt_2, c(generic_all_nodes), list_with_end, rule(small_stmt), tok(DEL_SEMICOLON)) // small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt @@ -99,16 +99,16 @@ DEF_RULE(simple_stmt_2, c(generic_all_nodes), list_with_end, rule(small_stmt), t // augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' // # For normal assignments, additional restrictions enforced by the interpreter -DEF_RULE(small_stmt, nc, or(8), rule(del_stmt), rule(pass_stmt), rule(flow_stmt), rule(import_stmt), rule(global_stmt), rule(nonlocal_stmt), rule(assert_stmt), rule(expr_stmt)) +DEF_RULE_NC(small_stmt, or(8), rule(del_stmt), rule(pass_stmt), rule(flow_stmt), rule(import_stmt), rule(global_stmt), rule(nonlocal_stmt), rule(assert_stmt), rule(expr_stmt)) DEF_RULE(expr_stmt, c(expr_stmt), and(2), rule(testlist_star_expr), opt_rule(expr_stmt_2)) -DEF_RULE(expr_stmt_2, nc, or(2), rule(expr_stmt_augassign), rule(expr_stmt_assign_list)) -DEF_RULE(expr_stmt_augassign, nc, and_ident(2), rule(augassign), rule(expr_stmt_6)) -DEF_RULE(expr_stmt_assign_list, nc, one_or_more, rule(expr_stmt_assign)) -DEF_RULE(expr_stmt_assign, nc, and_ident(2), tok(DEL_EQUAL), rule(expr_stmt_6)) -DEF_RULE(expr_stmt_6, nc, or(2), rule(yield_expr), rule(testlist_star_expr)) +DEF_RULE_NC(expr_stmt_2, or(2), rule(expr_stmt_augassign), rule(expr_stmt_assign_list)) +DEF_RULE_NC(expr_stmt_augassign, and_ident(2), rule(augassign), rule(expr_stmt_6)) +DEF_RULE_NC(expr_stmt_assign_list, one_or_more, rule(expr_stmt_assign)) +DEF_RULE_NC(expr_stmt_assign, and_ident(2), tok(DEL_EQUAL), rule(expr_stmt_6)) +DEF_RULE_NC(expr_stmt_6, or(2), rule(yield_expr), rule(testlist_star_expr)) DEF_RULE(testlist_star_expr, c(generic_tuple), list_with_end, rule(testlist_star_expr_2), tok(DEL_COMMA)) -DEF_RULE(testlist_star_expr_2, nc, or(2), rule(star_expr), rule(test)) -DEF_RULE(augassign, nc, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL)) +DEF_RULE_NC(testlist_star_expr_2, or(2), rule(star_expr), rule(test)) +DEF_RULE_NC(augassign, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL)) // del_stmt: 'del' exprlist // pass_stmt: 'pass' @@ -121,14 +121,14 @@ DEF_RULE(augassign, nc, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(D DEF_RULE(del_stmt, c(del_stmt), and(2), tok(KW_DEL), rule(exprlist)) DEF_RULE(pass_stmt, c(generic_all_nodes), and(1), tok(KW_PASS)) -DEF_RULE(flow_stmt, nc, or(5), rule(break_stmt), rule(continue_stmt), rule(return_stmt), rule(raise_stmt), rule(yield_stmt)) +DEF_RULE_NC(flow_stmt, or(5), rule(break_stmt), rule(continue_stmt), rule(return_stmt), rule(raise_stmt), rule(yield_stmt)) DEF_RULE(break_stmt, c(break_stmt), and(1), tok(KW_BREAK)) DEF_RULE(continue_stmt, c(continue_stmt), and(1), tok(KW_CONTINUE)) DEF_RULE(return_stmt, c(return_stmt), and(2), tok(KW_RETURN), opt_rule(testlist)) DEF_RULE(yield_stmt, c(yield_stmt), and(1), rule(yield_expr)) DEF_RULE(raise_stmt, c(raise_stmt), and(2), tok(KW_RAISE), opt_rule(raise_stmt_arg)) -DEF_RULE(raise_stmt_arg, nc, and_ident(2), rule(test), opt_rule(raise_stmt_from)) -DEF_RULE(raise_stmt_from, nc, and_ident(2), tok(KW_FROM), rule(test)) +DEF_RULE_NC(raise_stmt_arg, and_ident(2), rule(test), opt_rule(raise_stmt_from)) +DEF_RULE_NC(raise_stmt_from, and_ident(2), tok(KW_FROM), rule(test)) // import_stmt: import_name | import_from // import_name: 'import' dotted_as_names @@ -142,26 +142,26 @@ DEF_RULE(raise_stmt_from, nc, and_ident(2), tok(KW_FROM), rule(test)) // nonlocal_stmt: 'nonlocal' NAME (',' NAME)* // assert_stmt: 'assert' test [',' test] -DEF_RULE(import_stmt, nc, or(2), rule(import_name), rule(import_from)) +DEF_RULE_NC(import_stmt, or(2), rule(import_name), rule(import_from)) DEF_RULE(import_name, c(import_name), and(2), tok(KW_IMPORT), rule(dotted_as_names)) DEF_RULE(import_from, c(import_from), and(4), tok(KW_FROM), rule(import_from_2), tok(KW_IMPORT), rule(import_from_3)) -DEF_RULE(import_from_2, nc, or(2), rule(dotted_name), rule(import_from_2b)) -DEF_RULE(import_from_2b, nc, and_ident(2), rule(one_or_more_period_or_ellipsis), opt_rule(dotted_name)) -DEF_RULE(import_from_3, nc, or(3), tok(OP_STAR), rule(import_as_names_paren), rule(import_as_names)) -DEF_RULE(import_as_names_paren, nc, and_ident(3), tok(DEL_PAREN_OPEN), rule(import_as_names), tok(DEL_PAREN_CLOSE)) -DEF_RULE(one_or_more_period_or_ellipsis, nc, one_or_more, rule(period_or_ellipsis)) -DEF_RULE(period_or_ellipsis, nc, or(2), tok(DEL_PERIOD), tok(ELLIPSIS)) -DEF_RULE(import_as_name, nc, and(2), tok(NAME), opt_rule(as_name)) -DEF_RULE(dotted_as_name, nc, and_ident(2), rule(dotted_name), opt_rule(as_name)) -DEF_RULE(as_name, nc, and_ident(2), tok(KW_AS), tok(NAME)) -DEF_RULE(import_as_names, nc, list_with_end, rule(import_as_name), tok(DEL_COMMA)) -DEF_RULE(dotted_as_names, nc, list, rule(dotted_as_name), tok(DEL_COMMA)) -DEF_RULE(dotted_name, nc, list, tok(NAME), tok(DEL_PERIOD)) +DEF_RULE_NC(import_from_2, or(2), rule(dotted_name), rule(import_from_2b)) +DEF_RULE_NC(import_from_2b, and_ident(2), rule(one_or_more_period_or_ellipsis), opt_rule(dotted_name)) +DEF_RULE_NC(import_from_3, or(3), tok(OP_STAR), rule(import_as_names_paren), rule(import_as_names)) +DEF_RULE_NC(import_as_names_paren, and_ident(3), tok(DEL_PAREN_OPEN), rule(import_as_names), tok(DEL_PAREN_CLOSE)) +DEF_RULE_NC(one_or_more_period_or_ellipsis, one_or_more, rule(period_or_ellipsis)) +DEF_RULE_NC(period_or_ellipsis, or(2), tok(DEL_PERIOD), tok(ELLIPSIS)) +DEF_RULE_NC(import_as_name, and(2), tok(NAME), opt_rule(as_name)) +DEF_RULE_NC(dotted_as_name, and_ident(2), rule(dotted_name), opt_rule(as_name)) +DEF_RULE_NC(as_name, and_ident(2), tok(KW_AS), tok(NAME)) +DEF_RULE_NC(import_as_names, list_with_end, rule(import_as_name), tok(DEL_COMMA)) +DEF_RULE_NC(dotted_as_names, list, rule(dotted_as_name), tok(DEL_COMMA)) +DEF_RULE_NC(dotted_name, list, tok(NAME), tok(DEL_PERIOD)) DEF_RULE(global_stmt, c(global_stmt), and(2), tok(KW_GLOBAL), rule(name_list)) DEF_RULE(nonlocal_stmt, c(nonlocal_stmt), and(2), tok(KW_NONLOCAL), rule(name_list)) -DEF_RULE(name_list, nc, list, tok(NAME), tok(DEL_COMMA)) +DEF_RULE_NC(name_list, list, tok(NAME), tok(DEL_COMMA)) DEF_RULE(assert_stmt, c(assert_stmt), and(3), tok(KW_ASSERT), rule(test), opt_rule(assert_stmt_extra)) -DEF_RULE(assert_stmt_extra, nc, and_ident(2), tok(DEL_COMMA), rule(test)) +DEF_RULE_NC(assert_stmt_extra, and_ident(2), tok(DEL_COMMA), rule(test)) // compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt // if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] @@ -176,31 +176,31 @@ DEF_RULE(assert_stmt_extra, nc, and_ident(2), tok(DEL_COMMA), rule(test)) // async_stmt: 'async' (funcdef | with_stmt | for_stmt) #if MICROPY_PY_ASYNC_AWAIT -DEF_RULE(compound_stmt, nc, or(9), rule(if_stmt), rule(while_stmt), rule(for_stmt), rule(try_stmt), rule(with_stmt), rule(funcdef), rule(classdef), rule(decorated), rule(async_stmt)) +DEF_RULE_NC(compound_stmt, or(9), rule(if_stmt), rule(while_stmt), rule(for_stmt), rule(try_stmt), rule(with_stmt), rule(funcdef), rule(classdef), rule(decorated), rule(async_stmt)) DEF_RULE(async_stmt, c(async_stmt), and(2), tok(KW_ASYNC), rule(async_stmt_2)) -DEF_RULE(async_stmt_2, nc, or(3), rule(funcdef), rule(with_stmt), rule(for_stmt)) +DEF_RULE_NC(async_stmt_2, or(3), rule(funcdef), rule(with_stmt), rule(for_stmt)) #else -DEF_RULE(compound_stmt, nc, or(8), rule(if_stmt), rule(while_stmt), rule(for_stmt), rule(try_stmt), rule(with_stmt), rule(funcdef), rule(classdef), rule(decorated)) +DEF_RULE_NC(compound_stmt, or(8), rule(if_stmt), rule(while_stmt), rule(for_stmt), rule(try_stmt), rule(with_stmt), rule(funcdef), rule(classdef), rule(decorated)) #endif DEF_RULE(if_stmt, c(if_stmt), and(6), tok(KW_IF), rule(test), tok(DEL_COLON), rule(suite), opt_rule(if_stmt_elif_list), opt_rule(else_stmt)) -DEF_RULE(if_stmt_elif_list, nc, one_or_more, rule(if_stmt_elif)) -DEF_RULE(if_stmt_elif, nc, and(4), tok(KW_ELIF), rule(test), tok(DEL_COLON), rule(suite)) +DEF_RULE_NC(if_stmt_elif_list, one_or_more, rule(if_stmt_elif)) +DEF_RULE_NC(if_stmt_elif, and(4), tok(KW_ELIF), rule(test), tok(DEL_COLON), rule(suite)) DEF_RULE(while_stmt, c(while_stmt), and(5), tok(KW_WHILE), rule(test), tok(DEL_COLON), rule(suite), opt_rule(else_stmt)) DEF_RULE(for_stmt, c(for_stmt), and(7), tok(KW_FOR), rule(exprlist), tok(KW_IN), rule(testlist), tok(DEL_COLON), rule(suite), opt_rule(else_stmt)) DEF_RULE(try_stmt, c(try_stmt), and(4), tok(KW_TRY), tok(DEL_COLON), rule(suite), rule(try_stmt_2)) -DEF_RULE(try_stmt_2, nc, or(2), rule(try_stmt_except_and_more), rule(try_stmt_finally)) -DEF_RULE(try_stmt_except_and_more, nc, and_ident(3), rule(try_stmt_except_list), opt_rule(else_stmt), opt_rule(try_stmt_finally)) -DEF_RULE(try_stmt_except, nc, and(4), tok(KW_EXCEPT), opt_rule(try_stmt_as_name), tok(DEL_COLON), rule(suite)) -DEF_RULE(try_stmt_as_name, nc, and_ident(2), rule(test), opt_rule(as_name)) -DEF_RULE(try_stmt_except_list, nc, one_or_more, rule(try_stmt_except)) -DEF_RULE(try_stmt_finally, nc, and(3), tok(KW_FINALLY), tok(DEL_COLON), rule(suite)) -DEF_RULE(else_stmt, nc, and_ident(3), tok(KW_ELSE), tok(DEL_COLON), rule(suite)) +DEF_RULE_NC(try_stmt_2, or(2), rule(try_stmt_except_and_more), rule(try_stmt_finally)) +DEF_RULE_NC(try_stmt_except_and_more, and_ident(3), rule(try_stmt_except_list), opt_rule(else_stmt), opt_rule(try_stmt_finally)) +DEF_RULE_NC(try_stmt_except, and(4), tok(KW_EXCEPT), opt_rule(try_stmt_as_name), tok(DEL_COLON), rule(suite)) +DEF_RULE_NC(try_stmt_as_name, and_ident(2), rule(test), opt_rule(as_name)) +DEF_RULE_NC(try_stmt_except_list, one_or_more, rule(try_stmt_except)) +DEF_RULE_NC(try_stmt_finally, and(3), tok(KW_FINALLY), tok(DEL_COLON), rule(suite)) +DEF_RULE_NC(else_stmt, and_ident(3), tok(KW_ELSE), tok(DEL_COLON), rule(suite)) DEF_RULE(with_stmt, c(with_stmt), and(4), tok(KW_WITH), rule(with_stmt_list), tok(DEL_COLON), rule(suite)) -DEF_RULE(with_stmt_list, nc, list, rule(with_item), tok(DEL_COMMA)) -DEF_RULE(with_item, nc, and_ident(2), rule(test), opt_rule(with_item_as)) -DEF_RULE(with_item_as, nc, and_ident(2), tok(KW_AS), rule(expr)) -DEF_RULE(suite, nc, or(2), rule(suite_block), rule(simple_stmt)) -DEF_RULE(suite_block, nc, and_ident(4), tok(NEWLINE), tok(INDENT), rule(suite_block_stmts), tok(DEDENT)) +DEF_RULE_NC(with_stmt_list, list, rule(with_item), tok(DEL_COMMA)) +DEF_RULE_NC(with_item, and_ident(2), rule(test), opt_rule(with_item_as)) +DEF_RULE_NC(with_item_as, and_ident(2), tok(KW_AS), rule(expr)) +DEF_RULE_NC(suite, or(2), rule(suite_block), rule(simple_stmt)) +DEF_RULE_NC(suite_block, and_ident(4), tok(NEWLINE), tok(INDENT), rule(suite_block_stmts), tok(DEDENT)) DEF_RULE(suite_block_stmts, c(generic_all_nodes), one_or_more, rule(stmt)) // test: or_test ['if' or_test 'else' test] | lambdef @@ -208,10 +208,10 @@ DEF_RULE(suite_block_stmts, c(generic_all_nodes), one_or_more, rule(stmt)) // lambdef: 'lambda' [varargslist] ':' test // lambdef_nocond: 'lambda' [varargslist] ':' test_nocond -DEF_RULE(test, nc, or(2), rule(lambdef), rule(test_if_expr)) +DEF_RULE_NC(test, or(2), rule(lambdef), rule(test_if_expr)) DEF_RULE(test_if_expr, c(test_if_expr), and_ident(2), rule(or_test), opt_rule(test_if_else)) -DEF_RULE(test_if_else, nc, and(4), tok(KW_IF), rule(or_test), tok(KW_ELSE), rule(test)) -DEF_RULE(test_nocond, nc, or(2), rule(lambdef_nocond), rule(or_test)) +DEF_RULE_NC(test_if_else, and(4), tok(KW_IF), rule(or_test), tok(KW_ELSE), rule(test)) +DEF_RULE_NC(test_nocond, or(2), rule(lambdef_nocond), rule(or_test)) DEF_RULE(lambdef, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(varargslist), tok(DEL_COLON), rule(test)) DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(varargslist), tok(DEL_COLON), rule(test_nocond)) @@ -233,54 +233,54 @@ DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(vara DEF_RULE(or_test, c(or_test), list, rule(and_test), tok(KW_OR)) DEF_RULE(and_test, c(and_test), list, rule(not_test), tok(KW_AND)) -DEF_RULE(not_test, nc, or(2), rule(not_test_2), rule(comparison)) +DEF_RULE_NC(not_test, or(2), rule(not_test_2), rule(comparison)) DEF_RULE(not_test_2, c(not_test_2), and(2), tok(KW_NOT), rule(not_test)) DEF_RULE(comparison, c(comparison), list, rule(expr), rule(comp_op)) -DEF_RULE(comp_op, nc, or(9), tok(OP_LESS), tok(OP_MORE), tok(OP_DBL_EQUAL), tok(OP_LESS_EQUAL), tok(OP_MORE_EQUAL), tok(OP_NOT_EQUAL), tok(KW_IN), rule(comp_op_not_in), rule(comp_op_is)) -DEF_RULE(comp_op_not_in, nc, and(2), tok(KW_NOT), tok(KW_IN)) -DEF_RULE(comp_op_is, nc, and(2), tok(KW_IS), opt_rule(comp_op_is_not)) -DEF_RULE(comp_op_is_not, nc, and(1), tok(KW_NOT)) +DEF_RULE_NC(comp_op, or(9), tok(OP_LESS), tok(OP_MORE), tok(OP_DBL_EQUAL), tok(OP_LESS_EQUAL), tok(OP_MORE_EQUAL), tok(OP_NOT_EQUAL), tok(KW_IN), rule(comp_op_not_in), rule(comp_op_is)) +DEF_RULE_NC(comp_op_not_in, and(2), tok(KW_NOT), tok(KW_IN)) +DEF_RULE_NC(comp_op_is, and(2), tok(KW_IS), opt_rule(comp_op_is_not)) +DEF_RULE_NC(comp_op_is_not, and(1), tok(KW_NOT)) DEF_RULE(star_expr, c(star_expr), and(2), tok(OP_STAR), rule(expr)) DEF_RULE(expr, c(expr), list, rule(xor_expr), tok(OP_PIPE)) DEF_RULE(xor_expr, c(xor_expr), list, rule(and_expr), tok(OP_CARET)) DEF_RULE(and_expr, c(and_expr), list, rule(shift_expr), tok(OP_AMPERSAND)) DEF_RULE(shift_expr, c(shift_expr), list, rule(arith_expr), rule(shift_op)) -DEF_RULE(shift_op, nc, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE)) +DEF_RULE_NC(shift_op, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE)) DEF_RULE(arith_expr, c(arith_expr), list, rule(term), rule(arith_op)) -DEF_RULE(arith_op, nc, or(2), tok(OP_PLUS), tok(OP_MINUS)) +DEF_RULE_NC(arith_op, or(2), tok(OP_PLUS), tok(OP_MINUS)) DEF_RULE(term, c(term), list, rule(factor), rule(term_op)) -DEF_RULE(term_op, nc, or(4), tok(OP_STAR), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH)) -DEF_RULE(factor, nc, or(2), rule(factor_2), rule(power)) +DEF_RULE_NC(term_op, or(4), tok(OP_STAR), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH)) +DEF_RULE_NC(factor, or(2), rule(factor_2), rule(power)) DEF_RULE(factor_2, c(factor_2), and_ident(2), rule(factor_op), rule(factor)) -DEF_RULE(factor_op, nc, or(3), tok(OP_PLUS), tok(OP_MINUS), tok(OP_TILDE)) +DEF_RULE_NC(factor_op, or(3), tok(OP_PLUS), tok(OP_MINUS), tok(OP_TILDE)) DEF_RULE(power, c(power), and_ident(2), rule(atom_expr), opt_rule(power_dbl_star)) #if MICROPY_PY_ASYNC_AWAIT -DEF_RULE(atom_expr, nc, or(2), rule(atom_expr_await), rule(atom_expr_normal)) +DEF_RULE_NC(atom_expr, or(2), rule(atom_expr_await), rule(atom_expr_normal)) DEF_RULE(atom_expr_await, c(atom_expr_await), and(3), tok(KW_AWAIT), rule(atom), opt_rule(atom_expr_trailers)) #else -DEF_RULE(atom_expr, nc, or(1), rule(atom_expr_normal)) +DEF_RULE_NC(atom_expr, or(1), rule(atom_expr_normal)) #endif DEF_RULE(atom_expr_normal, c(atom_expr_normal), and_ident(2), rule(atom), opt_rule(atom_expr_trailers)) DEF_RULE(atom_expr_trailers, c(atom_expr_trailers), one_or_more, rule(trailer)) -DEF_RULE(power_dbl_star, nc, and_ident(2), tok(OP_DBL_STAR), rule(factor)) +DEF_RULE_NC(power_dbl_star, and_ident(2), tok(OP_DBL_STAR), rule(factor)) // atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' | '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False' // testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) // trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -DEF_RULE(atom, nc, or(11), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), rule(atom_string), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) +DEF_RULE_NC(atom, or(11), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), rule(atom_string), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) DEF_RULE(atom_string, c(atom_string), one_or_more, rule(string_or_bytes)) -DEF_RULE(string_or_bytes, nc, or(2), tok(STRING), tok(BYTES)) +DEF_RULE_NC(string_or_bytes, or(2), tok(STRING), tok(BYTES)) DEF_RULE(atom_paren, c(atom_paren), and(3), tok(DEL_PAREN_OPEN), opt_rule(atom_2b), tok(DEL_PAREN_CLOSE)) -DEF_RULE(atom_2b, nc, or(2), rule(yield_expr), rule(testlist_comp)) +DEF_RULE_NC(atom_2b, or(2), rule(yield_expr), rule(testlist_comp)) DEF_RULE(atom_bracket, c(atom_bracket), and(3), tok(DEL_BRACKET_OPEN), opt_rule(testlist_comp), tok(DEL_BRACKET_CLOSE)) DEF_RULE(atom_brace, c(atom_brace), and(3), tok(DEL_BRACE_OPEN), opt_rule(dictorsetmaker), tok(DEL_BRACE_CLOSE)) -DEF_RULE(testlist_comp, nc, and_ident(2), rule(testlist_comp_2), opt_rule(testlist_comp_3)) -DEF_RULE(testlist_comp_2, nc, or(2), rule(star_expr), rule(test)) -DEF_RULE(testlist_comp_3, nc, or(2), rule(comp_for), rule(testlist_comp_3b)) -DEF_RULE(testlist_comp_3b, nc, and_ident(2), tok(DEL_COMMA), opt_rule(testlist_comp_3c)) -DEF_RULE(testlist_comp_3c, nc, list_with_end, rule(testlist_comp_2), tok(DEL_COMMA)) -DEF_RULE(trailer, nc, or(3), rule(trailer_paren), rule(trailer_bracket), rule(trailer_period)) +DEF_RULE_NC(testlist_comp, and_ident(2), rule(testlist_comp_2), opt_rule(testlist_comp_3)) +DEF_RULE_NC(testlist_comp_2, or(2), rule(star_expr), rule(test)) +DEF_RULE_NC(testlist_comp_3, or(2), rule(comp_for), rule(testlist_comp_3b)) +DEF_RULE_NC(testlist_comp_3b, and_ident(2), tok(DEL_COMMA), opt_rule(testlist_comp_3c)) +DEF_RULE_NC(testlist_comp_3c, list_with_end, rule(testlist_comp_2), tok(DEL_COMMA)) +DEF_RULE_NC(trailer, or(3), rule(trailer_paren), rule(trailer_bracket), rule(trailer_period)) DEF_RULE(trailer_paren, c(trailer_paren), and(3), tok(DEL_PAREN_OPEN), opt_rule(arglist), tok(DEL_PAREN_CLOSE)) DEF_RULE(trailer_bracket, c(trailer_bracket), and(3), tok(DEL_BRACKET_OPEN), rule(subscriptlist), tok(DEL_BRACKET_CLOSE)) DEF_RULE(trailer_period, c(trailer_period), and(2), tok(DEL_PERIOD), tok(NAME)) @@ -291,13 +291,13 @@ DEF_RULE(trailer_period, c(trailer_period), and(2), tok(DEL_PERIOD), tok(NAME)) #if MICROPY_PY_BUILTINS_SLICE DEF_RULE(subscriptlist, c(generic_tuple), list_with_end, rule(subscript), tok(DEL_COMMA)) -DEF_RULE(subscript, nc, or(2), rule(subscript_3), rule(subscript_2)) +DEF_RULE_NC(subscript, or(2), rule(subscript_3), rule(subscript_2)) DEF_RULE(subscript_2, c(subscript_2), and_ident(2), rule(test), opt_rule(subscript_3)) DEF_RULE(subscript_3, c(subscript_3), and(2), tok(DEL_COLON), opt_rule(subscript_3b)) -DEF_RULE(subscript_3b, nc, or(2), rule(subscript_3c), rule(subscript_3d)) -DEF_RULE(subscript_3c, nc, and(2), tok(DEL_COLON), opt_rule(test)) -DEF_RULE(subscript_3d, nc, and_ident(2), rule(test), opt_rule(sliceop)) -DEF_RULE(sliceop, nc, and(2), tok(DEL_COLON), opt_rule(test)) +DEF_RULE_NC(subscript_3b, or(2), rule(subscript_3c), rule(subscript_3d)) +DEF_RULE_NC(subscript_3c, and(2), tok(DEL_COLON), opt_rule(test)) +DEF_RULE_NC(subscript_3d, and_ident(2), rule(test), opt_rule(sliceop)) +DEF_RULE_NC(sliceop, and(2), tok(DEL_COLON), opt_rule(test)) #else DEF_RULE(subscriptlist, c(generic_tuple), list_with_end, rule(test), tok(DEL_COMMA)) #endif @@ -306,33 +306,33 @@ DEF_RULE(subscriptlist, c(generic_tuple), list_with_end, rule(test), tok(DEL_COM // testlist: test (',' test)* [','] // dictorsetmaker: (test ':' test (comp_for | (',' test ':' test)* [','])) | (test (comp_for | (',' test)* [','])) -DEF_RULE(exprlist, nc, list_with_end, rule(exprlist_2), tok(DEL_COMMA)) -DEF_RULE(exprlist_2, nc, or(2), rule(star_expr), rule(expr)) +DEF_RULE_NC(exprlist, list_with_end, rule(exprlist_2), tok(DEL_COMMA)) +DEF_RULE_NC(exprlist_2, or(2), rule(star_expr), rule(expr)) DEF_RULE(testlist, c(generic_tuple), list_with_end, rule(test), tok(DEL_COMMA)) // TODO dictorsetmaker lets through more than is allowed -DEF_RULE(dictorsetmaker, nc, and_ident(2), rule(dictorsetmaker_item), opt_rule(dictorsetmaker_tail)) +DEF_RULE_NC(dictorsetmaker, and_ident(2), rule(dictorsetmaker_item), opt_rule(dictorsetmaker_tail)) #if MICROPY_PY_BUILTINS_SET DEF_RULE(dictorsetmaker_item, c(dictorsetmaker_item), and_ident(2), rule(test), opt_rule(dictorsetmaker_colon)) -DEF_RULE(dictorsetmaker_colon, nc, and_ident(2), tok(DEL_COLON), rule(test)) +DEF_RULE_NC(dictorsetmaker_colon, and_ident(2), tok(DEL_COLON), rule(test)) #else DEF_RULE(dictorsetmaker_item, c(dictorsetmaker_item), and(3), rule(test), tok(DEL_COLON), rule(test)) #endif -DEF_RULE(dictorsetmaker_tail, nc, or(2), rule(comp_for), rule(dictorsetmaker_list)) -DEF_RULE(dictorsetmaker_list, nc, and(2), tok(DEL_COMMA), opt_rule(dictorsetmaker_list2)) -DEF_RULE(dictorsetmaker_list2, nc, list_with_end, rule(dictorsetmaker_item), tok(DEL_COMMA)) +DEF_RULE_NC(dictorsetmaker_tail, or(2), rule(comp_for), rule(dictorsetmaker_list)) +DEF_RULE_NC(dictorsetmaker_list, and(2), tok(DEL_COMMA), opt_rule(dictorsetmaker_list2)) +DEF_RULE_NC(dictorsetmaker_list2, list_with_end, rule(dictorsetmaker_item), tok(DEL_COMMA)) // classdef: 'class' NAME ['(' [arglist] ')'] ':' suite DEF_RULE(classdef, c(classdef), and_blank(5), tok(KW_CLASS), tok(NAME), opt_rule(classdef_2), tok(DEL_COLON), rule(suite)) -DEF_RULE(classdef_2, nc, and_ident(3), tok(DEL_PAREN_OPEN), opt_rule(arglist), tok(DEL_PAREN_CLOSE)) +DEF_RULE_NC(classdef_2, and_ident(3), tok(DEL_PAREN_OPEN), opt_rule(arglist), tok(DEL_PAREN_CLOSE)) // arglist: (argument ',')* (argument [','] | '*' test (',' argument)* [',' '**' test] | '**' test) // TODO arglist lets through more than is allowed, compiler needs to do further verification -DEF_RULE(arglist, nc, list_with_end, rule(arglist_2), tok(DEL_COMMA)) -DEF_RULE(arglist_2, nc, or(3), rule(arglist_star), rule(arglist_dbl_star), rule(argument)) -DEF_RULE(arglist_star, nc, and(2), tok(OP_STAR), rule(test)) -DEF_RULE(arglist_dbl_star, nc, and(2), tok(OP_DBL_STAR), rule(test)) +DEF_RULE_NC(arglist, list_with_end, rule(arglist_2), tok(DEL_COMMA)) +DEF_RULE_NC(arglist_2, or(3), rule(arglist_star), rule(arglist_dbl_star), rule(argument)) +DEF_RULE_NC(arglist_star, and(2), tok(OP_STAR), rule(test)) +DEF_RULE_NC(arglist_dbl_star, and(2), tok(OP_DBL_STAR), rule(test)) // # The reason that keywords are test nodes instead of NAME is that using NAME // # results in an ambiguity. ast.c makes sure it's a NAME. @@ -341,12 +341,12 @@ DEF_RULE(arglist_dbl_star, nc, and(2), tok(OP_DBL_STAR), rule(test)) // comp_for: 'for' exprlist 'in' or_test [comp_iter] // comp_if: 'if' test_nocond [comp_iter] -DEF_RULE(argument, nc, and_ident(2), rule(test), opt_rule(argument_2)) -DEF_RULE(argument_2, nc, or(2), rule(comp_for), rule(argument_3)) -DEF_RULE(argument_3, nc, and_ident(2), tok(DEL_EQUAL), rule(test)) -DEF_RULE(comp_iter, nc, or(2), rule(comp_for), rule(comp_if)) -DEF_RULE(comp_for, nc, and_blank(5), tok(KW_FOR), rule(exprlist), tok(KW_IN), rule(or_test), opt_rule(comp_iter)) -DEF_RULE(comp_if, nc, and(3), tok(KW_IF), rule(test_nocond), opt_rule(comp_iter)) +DEF_RULE_NC(argument, and_ident(2), rule(test), opt_rule(argument_2)) +DEF_RULE_NC(argument_2, or(2), rule(comp_for), rule(argument_3)) +DEF_RULE_NC(argument_3, and_ident(2), tok(DEL_EQUAL), rule(test)) +DEF_RULE_NC(comp_iter, or(2), rule(comp_for), rule(comp_if)) +DEF_RULE_NC(comp_for, and_blank(5), tok(KW_FOR), rule(exprlist), tok(KW_IN), rule(or_test), opt_rule(comp_iter)) +DEF_RULE_NC(comp_if, and(3), tok(KW_IF), rule(test_nocond), opt_rule(comp_iter)) // # not used in grammar, but may appear in "node" passed from Parser to Compiler // encoding_decl: NAME @@ -355,5 +355,5 @@ DEF_RULE(comp_if, nc, and(3), tok(KW_IF), rule(test_nocond), opt_rule(comp_iter) // yield_arg: 'from' test | testlist DEF_RULE(yield_expr, c(yield_expr), and(2), tok(KW_YIELD), opt_rule(yield_arg)) -DEF_RULE(yield_arg, nc, or(2), rule(yield_arg_from), rule(testlist)) -DEF_RULE(yield_arg_from, nc, and(2), tok(KW_FROM), rule(test)) +DEF_RULE_NC(yield_arg, or(2), rule(yield_arg_from), rule(testlist)) +DEF_RULE_NC(yield_arg_from, and(2), tok(KW_FROM), rule(test)) diff --git a/py/parse.c b/py/parse.c index b1c2c19ea3..d15af41581 100644 --- a/py/parse.c +++ b/py/parse.c @@ -69,13 +69,22 @@ typedef struct _rule_t { } rule_t; enum { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) RULE_##rule, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE - RULE_maximum_number_of, +#undef DEF_RULE_NC RULE_string, // special node for non-interned string RULE_bytes, // special node for non-interned bytes RULE_const_object, // special node for a constant, generic Python object + +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_##rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC }; #define or(n) (RULE_ACT_OR | n) @@ -90,8 +99,10 @@ enum { #define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) #ifdef USE_RULE_NAME #define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; +#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; #else #define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; +#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; #endif #include "py/grammar.h" #undef or @@ -103,11 +114,25 @@ enum { #undef opt_rule #undef one_or_more #undef DEF_RULE +#undef DEF_RULE_NC STATIC const rule_t *const rules[] = { +// define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) &rule_##rule, +#define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE +#undef DEF_RULE_NC + NULL, // RULE_string + NULL, // RULE_bytes + NULL, // RULE_const_object + +// define rules without a compile function +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) &rule_##rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC }; typedef struct _rule_stack_t { @@ -215,7 +240,6 @@ STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, siz STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); size_t rule_id = arg & RULE_ARG_ARG_MASK; - assert(rule_id < RULE_maximum_number_of); push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0); } From 0ec957d7c519b01010e103197282933bfdcf0993 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 Feb 2017 11:55:31 +1100 Subject: [PATCH 72/92] tests/cmdline: Update cmd_parsetree test for changes to grammar order. --- tests/cmdline/cmd_parsetree.py.exp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index 5510d11cd0..17e4c9638c 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -1,28 +1,28 @@ ---------------- -[ 4] rule(2) (n=8) +[ 4] rule(1) (n=8) tok(5) -[ 4] rule(78) (n=4) +[ 4] rule(22) (n=4) id(i) -[ 4] rule(131) (n=1) +[ 4] rule(46) (n=1) NULL -[ 5] rule(42) (n=0) +[ 5] rule(8) (n=0) NULL -[ 6] rule(32) (n=2) +[ 6] rule(5) (n=2) id(a) tok(15) -[ 7] rule(32) (n=2) +[ 7] rule(5) (n=2) id(b) str(str) -[ 8] rule(32) (n=2) +[ 8] rule(5) (n=2) id(c) [ 8] literal str(a very long str that will not be interned) -[ 9] rule(32) (n=2) +[ 9] rule(5) (n=2) id(d) bytes(bytes) -[ 10] rule(32) (n=2) +[ 10] rule(5) (n=2) id(e) [ 10] literal bytes(a very long bytes that will not be interned) -[ 11] rule(32) (n=2) +[ 11] rule(5) (n=2) id(f) [ 11] literal \.\+ ---------------- From c26441474686a5fac93c82d9759b7ed91bed2978 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 16 Feb 2017 20:23:41 +1100 Subject: [PATCH 73/92] py/lexer: Don't generate string representation for period or ellipsis. It's not needed. --- py/lexer.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index 33af21e9c3..d89322ed7f 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -574,12 +574,9 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) { // special handling for . and ... operators, because .. is not a valid operator // get first char - vstr_add_char(&lex->vstr, '.'); next_char(lex); if (is_char_and(lex, '.', '.')) { - vstr_add_char(&lex->vstr, '.'); - vstr_add_char(&lex->vstr, '.'); next_char(lex); next_char(lex); lex->tok_kind = MP_TOKEN_ELLIPSIS; From 6a11048af1d01c78bdacddadd1b72dc7ba7c6478 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 00:19:34 +1100 Subject: [PATCH 74/92] py/persistentcode: Bump .mpy version due to change in bytecode. --- py/persistentcode.c | 13 ++++++++----- tools/mpy-tool.py | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/py/persistentcode.c b/py/persistentcode.c index 07395b304b..5cb5117093 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -38,6 +38,9 @@ #include "py/smallint.h" +// The current version of .mpy files +#define MPY_VERSION (1) + // The feature flags byte encodes the compile-time config options that // affect the generate bytecode. #define MPY_FEATURE_FLAGS ( \ @@ -209,10 +212,10 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { byte header[4]; read_bytes(reader, header, sizeof(header)); - if (strncmp((char*)header, "M\x00", 2) != 0) { - mp_raise_ValueError("invalid .mpy file"); - } - if (header[2] != MPY_FEATURE_FLAGS || header[3] > mp_small_int_bits()) { + if (header[0] != 'M' + || header[1] != MPY_VERSION + || header[2] != MPY_FEATURE_FLAGS + || header[3] > mp_small_int_bits()) { mp_raise_ValueError("incompatible .mpy file"); } mp_raw_code_t *rc = load_raw_code(reader); @@ -359,7 +362,7 @@ void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { // byte version // byte feature flags // byte number of bits in a small int - byte header[4] = {'M', 0, MPY_FEATURE_FLAGS_DYNAMIC, + byte header[4] = {'M', MPY_VERSION, MPY_FEATURE_FLAGS_DYNAMIC, #if MICROPY_DYNAMIC_COMPILER mp_dynamic_compiler.small_int_bits, #else diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index ce373a4f5b..d14e0f4ea4 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -57,6 +57,7 @@ class FreezeError(Exception): return 'error while freezing %s: %s' % (self.rawcode.source_file, self.msg) class Config: + MPY_VERSION = 1 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -438,8 +439,8 @@ def read_mpy(filename): header = bytes_cons(f.read(4)) if header[0] != ord('M'): raise Exception('not a valid .mpy file') - if header[1] != 0: - raise Exception('incompatible version') + if header[1] != config.MPY_VERSION: + raise Exception('incompatible .mpy version') feature_flags = header[2] config.MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE = (feature_flags & 1) != 0 config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_flags & 2) != 0 From 9214e39b3c43c344b125adbe71f374eb8f1a11c7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 10:23:14 +1100 Subject: [PATCH 75/92] gitattributes: Add .mpy files to list of binary files. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 85bc6fdb88..bc3a9abf9a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.png binary *.jpg binary *.dxf binary +*.mpy binary # These should also not be modified by git. tests/basics/string_cr_conversion.py -text From 30f3bcdd29644a92217d9e16d39b9571b52086b6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 10:27:34 +1100 Subject: [PATCH 76/92] gitattributes: Remove obsolete lines. --- .gitattributes | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index bc3a9abf9a..2d53b50e44 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,7 +15,6 @@ # These should also not be modified by git. tests/basics/string_cr_conversion.py -text tests/basics/string_crlf_conversion.py -text -stmhal/startup_stm32f40xx.s -text stmhal/pybcdc.inf_template -text stmhal/usbd_* -text stmhal/boards/*/stm32f4xx_hal_conf.h -text @@ -29,4 +28,3 @@ cc3200/hal/des.c -text cc3200/hal/i2s.c -text cc3200/hal/i2s.h -text cc3200/version.h -text -lib/fatfs/** -text From b0599de48e91afaaf33ec279197d170ddeb89483 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 10:27:47 +1100 Subject: [PATCH 77/92] minimal: Update frozentest.mpy file for new .mpy version. --- minimal/frozentest.mpy | Bin 255 -> 255 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/minimal/frozentest.mpy b/minimal/frozentest.mpy index c8345b1910611e890fde2e3058f3c41cf2c19970..5cb356d611968fc8169bb3758b365928ec5982f3 100644 GIT binary patch delta 90 zcmey*_@6Pzmyt<6hM55bc)1w9GiYgOXlQFNF#MM}&LH-KA%)Qh$PkCoj8HxkgcjNL YewHYZ*hM55bc%Lx*X3)~m(9qUkVE8Z7&LH-OA%)Qh$PkCoj8HxkgcjNL YewHYZ* Date: Fri, 17 Feb 2017 10:56:06 +1100 Subject: [PATCH 78/92] py/lexer: Simplify handling of indenting of very first token. --- py/lexer.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index d89322ed7f..91019e7e43 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -283,7 +283,7 @@ STATIC bool get_hex(mp_lexer_t *lex, mp_uint_t num_digits, mp_uint_t *result) { return true; } -STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) { +void mp_lexer_to_next(mp_lexer_t *lex) { // start new token text vstr_reset(&lex->vstr); @@ -322,14 +322,7 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) { lex->tok_line = lex->line; lex->tok_column = lex->column; - if (first_token && lex->line == 1 && lex->column != 1) { - // check that the first token is in the first column - // if first token is not on first line, we get a physical newline and - // this check is done as part of normal indent/dedent checking below - // (done to get equivalence with CPython) - lex->tok_kind = MP_TOKEN_INDENT; - - } else if (lex->emit_dent < 0) { + if (lex->emit_dent < 0) { lex->tok_kind = MP_TOKEN_DEDENT; lex->emit_dent += 1; @@ -705,7 +698,7 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { vstr_init(&lex->vstr, 32); // check for memory allocation error - // note: vstr_init above may fail on malloc, but so may mp_lexer_next_token_into below + // note: vstr_init above may fail on malloc, but so may mp_lexer_to_next below if (lex->indent_level == NULL) { mp_lexer_free(lex); return NULL; @@ -737,7 +730,13 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { } // preload first token - mp_lexer_next_token_into(lex, true); + mp_lexer_to_next(lex); + + // Check that the first token is in the first column. If it's not then we + // convert the token kind to INDENT so that the parser gives a syntax error. + if (lex->tok_column != 1) { + lex->tok_kind = MP_TOKEN_INDENT; + } return lex; } @@ -785,10 +784,6 @@ void mp_lexer_free(mp_lexer_t *lex) { } } -void mp_lexer_to_next(mp_lexer_t *lex) { - mp_lexer_next_token_into(lex, false); -} - #if 0 // This function is used to print the current token and should only be // needed to debug the lexer, so it's not available via a config option. From a68c75468812859633814e90207faec1a361778e Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 10:59:57 +1100 Subject: [PATCH 79/92] py/lexer: Move check for keyword to name-tokenising block. Keywords only needs to be searched for if the token is a MP_TOKEN_NAME, so we can move the seach to the part of the code that does the tokenising for MP_TOKEN_NAME. --- py/lexer.c | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index 91019e7e43..6a3fa656b1 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -527,6 +527,23 @@ void mp_lexer_to_next(mp_lexer_t *lex) { next_char(lex); } + // Check if the name is a keyword. + // We also check for __debug__ here and convert it to its value. This is + // so the parser gives a syntax error on, eg, x.__debug__. Otherwise, we + // need to check for this special token in many places in the compiler. + // TODO improve speed of these string comparisons + for (size_t i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) { + if (str_strn_equal(tok_kw[i], lex->vstr.buf, lex->vstr.len)) { + if (i == MP_ARRAY_SIZE(tok_kw) - 1) { + // tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__" + lex->tok_kind = (MP_STATE_VM(mp_optimise_value) == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE); + } else { + lex->tok_kind = MP_TOKEN_KW_FALSE + i; + } + break; + } + } + } else if (is_digit(lex) || (is_char(lex, '.') && is_following_digit(lex))) { bool forced_integer = false; if (is_char(lex, '.')) { @@ -655,26 +672,6 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } } } - - // check for keywords - if (lex->tok_kind == MP_TOKEN_NAME) { - // We check for __debug__ here and convert it to its value. This is so - // the parser gives a syntax error on, eg, x.__debug__. Otherwise, we - // need to check for this special token in many places in the compiler. - // TODO improve speed of these string comparisons - //for (mp_int_t i = 0; tok_kw[i] != NULL; i++) { - for (size_t i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) { - if (str_strn_equal(tok_kw[i], lex->vstr.buf, lex->vstr.len)) { - if (i == MP_ARRAY_SIZE(tok_kw) - 1) { - // tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__" - lex->tok_kind = (MP_STATE_VM(mp_optimise_value) == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE); - } else { - lex->tok_kind = MP_TOKEN_KW_FALSE + i; - } - break; - } - } - } } mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { From ae436797927c3c9f7ccdc25dd78af3dd279ca7ff Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 11:10:35 +1100 Subject: [PATCH 80/92] py/lexer: Use strcmp to make keyword searching more efficient. Since the table of keywords is sorted, we can use strcmp to do the search and stop part way through the search if the comparison is less-than. Because all tokens that are names are subject to this search, this optimisation will improve the overall speed of the lexer when processing a script. The change also decreases code size by a little bit because we now use strcmp instead of the custom str_strn_equal function. --- py/lexer.c | 31 +++++++++++-------------------- py/lexer.h | 17 +++++++++-------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index 6a3fa656b1..5c942f9344 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -25,6 +25,7 @@ */ #include +#include #include #include "py/mpstate.h" @@ -39,19 +40,6 @@ // TODO seems that CPython allows NULL byte in the input stream // don't know if that's intentional or not, but we don't allow it -// TODO replace with a call to a standard function -STATIC bool str_strn_equal(const char *str, const char *strn, mp_uint_t len) { - mp_uint_t i = 0; - - while (i < len && *str == *strn) { - ++i; - ++str; - ++strn; - } - - return i == len && *str == 0; -} - #define MP_LEXER_EOF ((unichar)MP_READER_EOF) #define CUR_CHAR(lex) ((lex)->chr0) @@ -225,10 +213,12 @@ STATIC const uint8_t tok_enc_kind[] = { }; // must have the same order as enum in lexer.h +// must be sorted according to strcmp STATIC const char *const tok_kw[] = { "False", "None", "True", + "__debug__", "and", "as", "assert", @@ -263,7 +253,6 @@ STATIC const char *const tok_kw[] = { "while", "with", "yield", - "__debug__", }; // This is called with CUR_CHAR() before first hex digit, and should return with @@ -531,16 +520,18 @@ void mp_lexer_to_next(mp_lexer_t *lex) { // We also check for __debug__ here and convert it to its value. This is // so the parser gives a syntax error on, eg, x.__debug__. Otherwise, we // need to check for this special token in many places in the compiler. - // TODO improve speed of these string comparisons + const char *s = vstr_null_terminated_str(&lex->vstr); for (size_t i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) { - if (str_strn_equal(tok_kw[i], lex->vstr.buf, lex->vstr.len)) { - if (i == MP_ARRAY_SIZE(tok_kw) - 1) { - // tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__" + int cmp = strcmp(s, tok_kw[i]); + if (cmp == 0) { + lex->tok_kind = MP_TOKEN_KW_FALSE + i; + if (lex->tok_kind == MP_TOKEN_KW___DEBUG__) { lex->tok_kind = (MP_STATE_VM(mp_optimise_value) == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE); - } else { - lex->tok_kind = MP_TOKEN_KW_FALSE + i; } break; + } else if (cmp < 0) { + // Table is sorted and comparison was less-than, so stop searching + break; } } diff --git a/py/lexer.h b/py/lexer.h index 32aef96266..d407192856 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -61,6 +61,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_KW_FALSE, // 14 MP_TOKEN_KW_NONE, MP_TOKEN_KW_TRUE, + MP_TOKEN_KW___DEBUG__, MP_TOKEN_KW_AND, MP_TOKEN_KW_AS, MP_TOKEN_KW_ASSERT, @@ -71,7 +72,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_KW_BREAK, MP_TOKEN_KW_CLASS, MP_TOKEN_KW_CONTINUE, - MP_TOKEN_KW_DEF, // 23 + MP_TOKEN_KW_DEF, MP_TOKEN_KW_DEL, MP_TOKEN_KW_ELIF, MP_TOKEN_KW_ELSE, @@ -81,7 +82,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_KW_FROM, MP_TOKEN_KW_GLOBAL, MP_TOKEN_KW_IF, - MP_TOKEN_KW_IMPORT, // 33 + MP_TOKEN_KW_IMPORT, MP_TOKEN_KW_IN, MP_TOKEN_KW_IS, MP_TOKEN_KW_LAMBDA, @@ -91,12 +92,12 @@ typedef enum _mp_token_kind_t { MP_TOKEN_KW_PASS, MP_TOKEN_KW_RAISE, MP_TOKEN_KW_RETURN, - MP_TOKEN_KW_TRY, // 43 + MP_TOKEN_KW_TRY, MP_TOKEN_KW_WHILE, MP_TOKEN_KW_WITH, MP_TOKEN_KW_YIELD, - MP_TOKEN_OP_PLUS, // 47 + MP_TOKEN_OP_PLUS, MP_TOKEN_OP_MINUS, MP_TOKEN_OP_STAR, MP_TOKEN_OP_DBL_STAR, @@ -106,7 +107,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_OP_LESS, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_OP_MORE, - MP_TOKEN_OP_DBL_MORE, // 57 + MP_TOKEN_OP_DBL_MORE, MP_TOKEN_OP_AMPERSAND, MP_TOKEN_OP_PIPE, MP_TOKEN_OP_CARET, @@ -116,7 +117,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_OP_DBL_EQUAL, MP_TOKEN_OP_NOT_EQUAL, - MP_TOKEN_DEL_PAREN_OPEN, // 66 + MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE, MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE, @@ -126,7 +127,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_PERIOD, MP_TOKEN_DEL_SEMICOLON, - MP_TOKEN_DEL_AT, // 76 + MP_TOKEN_DEL_AT, MP_TOKEN_DEL_EQUAL, MP_TOKEN_DEL_PLUS_EQUAL, MP_TOKEN_DEL_MINUS_EQUAL, @@ -136,7 +137,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_DEL_PERCENT_EQUAL, MP_TOKEN_DEL_AMPERSAND_EQUAL, MP_TOKEN_DEL_PIPE_EQUAL, - MP_TOKEN_DEL_CARET_EQUAL, // 86 + MP_TOKEN_DEL_CARET_EQUAL, MP_TOKEN_DEL_DBL_MORE_EQUAL, MP_TOKEN_DEL_DBL_LESS_EQUAL, MP_TOKEN_DEL_DBL_STAR_EQUAL, From 773278ec3030ea9ed809c5a248fde2278ce4b557 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 11:30:14 +1100 Subject: [PATCH 81/92] py/lexer: Simplify handling of line-continuation error. Previous to this patch there was an explicit check for errors with line continuation (where backslash was not immediately followed by a newline). But this check is not necessary: if there is an error then the remaining logic of the tokeniser will reject the backslash and correctly produce a syntax error. --- py/lexer.c | 14 +++----------- py/lexer.h | 13 ++++++------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index 5c942f9344..ad4fe3fcb8 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -290,18 +290,10 @@ void mp_lexer_to_next(mp_lexer_t *lex) { next_char(lex); } // had_physical_newline will be set on next loop - } else if (is_char(lex, '\\')) { - // backslash (outside string literals) must appear just before a physical newline + } else if (is_char_and(lex, '\\', '\n')) { + // line-continuation, so don't set had_physical_newline + next_char(lex); next_char(lex); - if (!is_physical_newline(lex)) { - // SyntaxError: unexpected character after line continuation character - lex->tok_line = lex->line; - lex->tok_column = lex->column; - lex->tok_kind = MP_TOKEN_BAD_LINE_CONTINUATION; - return; - } else { - next_char(lex); - } } else { break; } diff --git a/py/lexer.h b/py/lexer.h index d407192856..d5382fc6a3 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -39,18 +39,17 @@ */ typedef enum _mp_token_kind_t { - MP_TOKEN_END, // 0 + MP_TOKEN_END, MP_TOKEN_INVALID, MP_TOKEN_DEDENT_MISMATCH, MP_TOKEN_LONELY_STRING_OPEN, - MP_TOKEN_BAD_LINE_CONTINUATION, - MP_TOKEN_NEWLINE, // 5 - MP_TOKEN_INDENT, // 6 - MP_TOKEN_DEDENT, // 7 + MP_TOKEN_NEWLINE, + MP_TOKEN_INDENT, + MP_TOKEN_DEDENT, - MP_TOKEN_NAME, // 8 + MP_TOKEN_NAME, MP_TOKEN_INTEGER, MP_TOKEN_FLOAT_OR_IMAG, MP_TOKEN_STRING, @@ -58,7 +57,7 @@ typedef enum _mp_token_kind_t { MP_TOKEN_ELLIPSIS, - MP_TOKEN_KW_FALSE, // 14 + MP_TOKEN_KW_FALSE, MP_TOKEN_KW_NONE, MP_TOKEN_KW_TRUE, MP_TOKEN_KW___DEBUG__, From 534b7c368dc2af7720f3aaed0c936ef46d773957 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 12:12:40 +1100 Subject: [PATCH 82/92] py: Do adjacent str/bytes literal concatenation in lexer, not compiler. It's much more efficient in RAM and code size to do implicit literal string concatenation in the lexer, as opposed to the compiler. RAM usage is reduced because the concatenation can be done right away in the tokeniser by just accumulating the string/bytes literals into the lexer's vstr. Prior to this patch adjacent strings/bytes would create a parse tree (one node per string/bytes) and then in the compiler a whole new chunk of memory was allocated to store the concatenated string, which used more than double the memory compared to just accumulating in the lexer. This patch also significantly reduces code size: bare-arm: -204 minimal: -204 unix x64: -328 stmhal: -208 esp8266: -284 cc3200: -224 --- py/compile.c | 59 --------- py/grammar.h | 3 +- py/lexer.c | 363 ++++++++++++++++++++++++++++----------------------- 3 files changed, 203 insertions(+), 222 deletions(-) diff --git a/py/compile.c b/py/compile.c index 3fb45cd69c..15e757d464 100644 --- a/py/compile.c +++ b/py/compile.c @@ -2301,65 +2301,6 @@ STATIC void compile_atom_expr_trailers(compiler_t *comp, mp_parse_node_struct_t } } -STATIC void compile_atom_string(compiler_t *comp, mp_parse_node_struct_t *pns) { - // a list of strings - - // check type of list (string or bytes) and count total number of bytes - int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); - size_t n_bytes = 0; - int string_kind = MP_PARSE_NODE_NULL; - for (int i = 0; i < n; i++) { - int pn_kind; - if (MP_PARSE_NODE_IS_LEAF(pns->nodes[i])) { - pn_kind = MP_PARSE_NODE_LEAF_KIND(pns->nodes[i]); - assert(pn_kind == MP_PARSE_NODE_STRING || pn_kind == MP_PARSE_NODE_BYTES); - n_bytes += qstr_len(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])); - } else { - assert(MP_PARSE_NODE_IS_STRUCT(pns->nodes[i])); - mp_parse_node_struct_t *pns_string = (mp_parse_node_struct_t*)pns->nodes[i]; - if (MP_PARSE_NODE_STRUCT_KIND(pns_string) == PN_string) { - pn_kind = MP_PARSE_NODE_STRING; - } else { - assert(MP_PARSE_NODE_STRUCT_KIND(pns_string) == PN_bytes); - pn_kind = MP_PARSE_NODE_BYTES; - } - n_bytes += pns_string->nodes[1]; - } - if (i == 0) { - string_kind = pn_kind; - } else if (pn_kind != string_kind) { - compile_syntax_error(comp, (mp_parse_node_t)pns, "cannot mix bytes and nonbytes literals"); - return; - } - } - - // if we are not in the last pass, just load a dummy object - if (comp->pass != MP_PASS_EMIT) { - EMIT_ARG(load_const_obj, mp_const_none); - return; - } - - // concatenate string/bytes - vstr_t vstr; - vstr_init_len(&vstr, n_bytes); - byte *s_dest = (byte*)vstr.buf; - for (int i = 0; i < n; i++) { - if (MP_PARSE_NODE_IS_LEAF(pns->nodes[i])) { - size_t s_len; - const byte *s = qstr_data(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]), &s_len); - memcpy(s_dest, s, s_len); - s_dest += s_len; - } else { - mp_parse_node_struct_t *pns_string = (mp_parse_node_struct_t*)pns->nodes[i]; - memcpy(s_dest, (const char*)pns_string->nodes[0], pns_string->nodes[1]); - s_dest += pns_string->nodes[1]; - } - } - - // load the object - EMIT_ARG(load_const_obj, mp_obj_new_str_from_vstr(string_kind == MP_PARSE_NODE_STRING ? &mp_type_str : &mp_type_bytes, &vstr)); -} - // pns needs to have 2 nodes, first is lhs of comprehension, second is PN_comp_for node STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, scope_kind_t kind) { assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 2); diff --git a/py/grammar.h b/py/grammar.h index 98f6d16aee..7d75881e1c 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -268,8 +268,7 @@ DEF_RULE_NC(power_dbl_star, and_ident(2), tok(OP_DBL_STAR), rule(factor)) // testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) // trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -DEF_RULE_NC(atom, or(11), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), rule(atom_string), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) -DEF_RULE(atom_string, c(atom_string), one_or_more, rule(string_or_bytes)) +DEF_RULE_NC(atom, or(12), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), tok(STRING), tok(BYTES), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) DEF_RULE_NC(string_or_bytes, or(2), tok(STRING), tok(BYTES)) DEF_RULE(atom_paren, c(atom_paren), and(3), tok(DEL_PAREN_OPEN), opt_rule(atom_2b), tok(DEL_PAREN_CLOSE)) DEF_RULE_NC(atom_2b, or(2), rule(yield_expr), rule(testlist_comp)) diff --git a/py/lexer.c b/py/lexer.c index ad4fe3fcb8..329875ab06 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -63,11 +63,9 @@ STATIC bool is_char_or3(mp_lexer_t *lex, byte c1, byte c2, byte c3) { return lex->chr0 == c1 || lex->chr0 == c2 || lex->chr0 == c3; } -/* STATIC bool is_char_following(mp_lexer_t *lex, byte c) { return lex->chr1 == c; } -*/ STATIC bool is_char_following_or(mp_lexer_t *lex, byte c1, byte c2) { return lex->chr1 == c1 || lex->chr1 == c2; @@ -106,6 +104,13 @@ STATIC bool is_following_odigit(mp_lexer_t *lex) { return lex->chr1 >= '0' && lex->chr1 <= '7'; } +STATIC bool is_string_or_bytes(mp_lexer_t *lex) { + return is_char_or(lex, '\'', '\"') + || (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"')) + || ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) + && is_char_following_following_or(lex, '\'', '\"')); +} + // to easily parse utf-8 identifiers we allow any raw byte with high bit set STATIC bool is_head_of_identifier(mp_lexer_t *lex) { return is_letter(lex) || lex->chr0 == '_' || lex->chr0 >= 0x80; @@ -272,14 +277,144 @@ STATIC bool get_hex(mp_lexer_t *lex, mp_uint_t num_digits, mp_uint_t *result) { return true; } -void mp_lexer_to_next(mp_lexer_t *lex) { - // start new token text - vstr_reset(&lex->vstr); +STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw) { + // get first quoting character + char quote_char = '\''; + if (is_char(lex, '\"')) { + quote_char = '\"'; + } + next_char(lex); - // skip white space and comments + // work out if it's a single or triple quoted literal + size_t num_quotes; + if (is_char_and(lex, quote_char, quote_char)) { + // triple quotes + next_char(lex); + next_char(lex); + num_quotes = 3; + } else { + // single quotes + num_quotes = 1; + } + + size_t n_closing = 0; + while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { + if (is_char(lex, quote_char)) { + n_closing += 1; + vstr_add_char(&lex->vstr, CUR_CHAR(lex)); + } else { + n_closing = 0; + if (is_char(lex, '\\')) { + next_char(lex); + unichar c = CUR_CHAR(lex); + if (is_raw) { + // raw strings allow escaping of quotes, but the backslash is also emitted + vstr_add_char(&lex->vstr, '\\'); + } else { + switch (c) { + // note: "c" can never be MP_LEXER_EOF because next_char + // always inserts a newline at the end of the input stream + case '\n': c = MP_LEXER_EOF; break; // backslash escape the newline, just ignore it + case '\\': break; + case '\'': break; + case '"': break; + case 'a': c = 0x07; break; + case 'b': c = 0x08; break; + case 't': c = 0x09; break; + case 'n': c = 0x0a; break; + case 'v': c = 0x0b; break; + case 'f': c = 0x0c; break; + case 'r': c = 0x0d; break; + case 'u': + case 'U': + if (lex->tok_kind == MP_TOKEN_BYTES) { + // b'\u1234' == b'\\u1234' + vstr_add_char(&lex->vstr, '\\'); + break; + } + // Otherwise fall through. + case 'x': + { + mp_uint_t num = 0; + if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) { + // not enough hex chars for escape sequence + lex->tok_kind = MP_TOKEN_INVALID; + } + c = num; + break; + } + case 'N': + // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the + // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly + // 3MB of text; even gzip-compressed and with minimal structure, it'll take + // roughly half a meg of storage. This form of Unicode escape may be added + // later on, but it's definitely not a priority right now. -- CJA 20140607 + mp_not_implemented("unicode name escapes"); + break; + default: + if (c >= '0' && c <= '7') { + // Octal sequence, 1-3 chars + mp_uint_t digits = 3; + mp_uint_t num = c - '0'; + while (is_following_odigit(lex) && --digits != 0) { + next_char(lex); + num = num * 8 + (CUR_CHAR(lex) - '0'); + } + c = num; + } else { + // unrecognised escape character; CPython lets this through verbatim as '\' and then the character + vstr_add_char(&lex->vstr, '\\'); + } + break; + } + } + if (c != MP_LEXER_EOF) { + if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) { + if (c < 0x110000 && lex->tok_kind == MP_TOKEN_STRING) { + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && lex->tok_kind == MP_TOKEN_BYTES) { + vstr_add_byte(&lex->vstr, c); + } else { + // unicode character out of range + // this raises a generic SyntaxError; could provide more info + lex->tok_kind = MP_TOKEN_INVALID; + } + } else { + // without unicode everything is just added as an 8-bit byte + if (c < 0x100) { + vstr_add_byte(&lex->vstr, c); + } else { + // 8-bit character out of range + // this raises a generic SyntaxError; could provide more info + lex->tok_kind = MP_TOKEN_INVALID; + } + } + } + } else { + // Add the "character" as a byte so that we remain 8-bit clean. + // This way, strings are parsed correctly whether or not they contain utf-8 chars. + vstr_add_byte(&lex->vstr, CUR_CHAR(lex)); + } + } + next_char(lex); + } + + // check we got the required end quotes + if (n_closing < num_quotes) { + lex->tok_kind = MP_TOKEN_LONELY_STRING_OPEN; + } + + // cut off the end quotes from the token text + vstr_cut_tail_bytes(&lex->vstr, n_closing); +} + +STATIC bool skip_whitespace(mp_lexer_t *lex, bool stop_at_newline) { bool had_physical_newline = false; while (!is_end(lex)) { if (is_physical_newline(lex)) { + if (stop_at_newline && lex->nested_bracket_level == 0) { + break; + } had_physical_newline = true; next_char(lex); } else if (is_whitespace(lex)) { @@ -298,6 +433,15 @@ void mp_lexer_to_next(mp_lexer_t *lex) { break; } } + return had_physical_newline; +} + +void mp_lexer_to_next(mp_lexer_t *lex) { + // start new token text + vstr_reset(&lex->vstr); + + // skip white space and comments + bool had_physical_newline = skip_whitespace(lex, false); // set token source information lex->tok_line = lex->line; @@ -332,168 +476,65 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } else if (is_end(lex)) { lex->tok_kind = MP_TOKEN_END; - } else if (is_char_or(lex, '\'', '\"') - || (is_char_or3(lex, 'r', 'u', 'b') && is_char_following_or(lex, '\'', '\"')) - || ((is_char_and(lex, 'r', 'b') || is_char_and(lex, 'b', 'r')) && is_char_following_following_or(lex, '\'', '\"'))) { + } else if (is_string_or_bytes(lex)) { // a string or bytes literal - // parse type codes - bool is_raw = false; - bool is_bytes = false; - if (is_char(lex, 'u')) { - next_char(lex); - } else if (is_char(lex, 'b')) { - is_bytes = true; - next_char(lex); - if (is_char(lex, 'r')) { + // Python requires adjacent string/bytes literals to be automatically + // concatenated. We do it here in the tokeniser to make efficient use of RAM, + // because then the lexer's vstr can be used to accumulate the string literal, + // in contrast to creating a parse tree of strings and then joining them later + // in the compiler. It's also more compact in code size to do it here. + + // MP_TOKEN_END is used to indicate that this is the first string token + lex->tok_kind = MP_TOKEN_END; + + // Loop to accumulate string/bytes literals + do { + // parse type codes + bool is_raw = false; + mp_token_kind_t kind = MP_TOKEN_STRING; + int n_char = 0; + if (is_char(lex, 'u')) { + n_char = 1; + } else if (is_char(lex, 'b')) { + kind = MP_TOKEN_BYTES; + n_char = 1; + if (is_char_following(lex, 'r')) { + is_raw = true; + n_char = 2; + } + } else if (is_char(lex, 'r')) { is_raw = true; - next_char(lex); - } - } else if (is_char(lex, 'r')) { - is_raw = true; - next_char(lex); - if (is_char(lex, 'b')) { - is_bytes = true; - next_char(lex); - } - } - - // set token kind - if (is_bytes) { - lex->tok_kind = MP_TOKEN_BYTES; - } else { - lex->tok_kind = MP_TOKEN_STRING; - } - - // get first quoting character - char quote_char = '\''; - if (is_char(lex, '\"')) { - quote_char = '\"'; - } - next_char(lex); - - // work out if it's a single or triple quoted literal - mp_uint_t num_quotes; - if (is_char_and(lex, quote_char, quote_char)) { - // triple quotes - next_char(lex); - next_char(lex); - num_quotes = 3; - } else { - // single quotes - num_quotes = 1; - } - - // parse the literal - mp_uint_t n_closing = 0; - while (!is_end(lex) && (num_quotes > 1 || !is_char(lex, '\n')) && n_closing < num_quotes) { - if (is_char(lex, quote_char)) { - n_closing += 1; - vstr_add_char(&lex->vstr, CUR_CHAR(lex)); - } else { - n_closing = 0; - if (is_char(lex, '\\')) { - next_char(lex); - unichar c = CUR_CHAR(lex); - if (is_raw) { - // raw strings allow escaping of quotes, but the backslash is also emitted - vstr_add_char(&lex->vstr, '\\'); - } else { - switch (c) { - // note: "c" can never be MP_LEXER_EOF because next_char - // always inserts a newline at the end of the input stream - case '\n': c = MP_LEXER_EOF; break; // backslash escape the newline, just ignore it - case '\\': break; - case '\'': break; - case '"': break; - case 'a': c = 0x07; break; - case 'b': c = 0x08; break; - case 't': c = 0x09; break; - case 'n': c = 0x0a; break; - case 'v': c = 0x0b; break; - case 'f': c = 0x0c; break; - case 'r': c = 0x0d; break; - case 'u': - case 'U': - if (is_bytes) { - // b'\u1234' == b'\\u1234' - vstr_add_char(&lex->vstr, '\\'); - break; - } - // Otherwise fall through. - case 'x': - { - mp_uint_t num = 0; - if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) { - // not enough hex chars for escape sequence - lex->tok_kind = MP_TOKEN_INVALID; - } - c = num; - break; - } - case 'N': - // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the - // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly - // 3MB of text; even gzip-compressed and with minimal structure, it'll take - // roughly half a meg of storage. This form of Unicode escape may be added - // later on, but it's definitely not a priority right now. -- CJA 20140607 - mp_not_implemented("unicode name escapes"); - break; - default: - if (c >= '0' && c <= '7') { - // Octal sequence, 1-3 chars - mp_uint_t digits = 3; - mp_uint_t num = c - '0'; - while (is_following_odigit(lex) && --digits != 0) { - next_char(lex); - num = num * 8 + (CUR_CHAR(lex) - '0'); - } - c = num; - } else { - // unrecognised escape character; CPython lets this through verbatim as '\' and then the character - vstr_add_char(&lex->vstr, '\\'); - } - break; - } - } - if (c != MP_LEXER_EOF) { - if (MICROPY_PY_BUILTINS_STR_UNICODE_DYNAMIC) { - if (c < 0x110000 && !is_bytes) { - vstr_add_char(&lex->vstr, c); - } else if (c < 0x100 && is_bytes) { - vstr_add_byte(&lex->vstr, c); - } else { - // unicode character out of range - // this raises a generic SyntaxError; could provide more info - lex->tok_kind = MP_TOKEN_INVALID; - } - } else { - // without unicode everything is just added as an 8-bit byte - if (c < 0x100) { - vstr_add_byte(&lex->vstr, c); - } else { - // 8-bit character out of range - // this raises a generic SyntaxError; could provide more info - lex->tok_kind = MP_TOKEN_INVALID; - } - } - } - } else { - // Add the "character" as a byte so that we remain 8-bit clean. - // This way, strings are parsed correctly whether or not they contain utf-8 chars. - vstr_add_byte(&lex->vstr, CUR_CHAR(lex)); + n_char = 1; + if (is_char_following(lex, 'b')) { + kind = MP_TOKEN_BYTES; + n_char = 2; } } - next_char(lex); - } - // check we got the required end quotes - if (n_closing < num_quotes) { - lex->tok_kind = MP_TOKEN_LONELY_STRING_OPEN; - } + // Set or check token kind + if (lex->tok_kind == MP_TOKEN_END) { + lex->tok_kind = kind; + } else if (lex->tok_kind != kind) { + // Can't concatenate string with bytes + break; + } - // cut off the end quotes from the token text - vstr_cut_tail_bytes(&lex->vstr, n_closing); + // Skip any type code characters + if (n_char != 0) { + next_char(lex); + if (n_char == 2) { + next_char(lex); + } + } + + // Parse the literal + parse_string_literal(lex, is_raw); + + // Skip whitespace so we can check if there's another string following + skip_whitespace(lex, true); + + } while (is_string_or_bytes(lex)); } else if (is_head_of_identifier(lex)) { lex->tok_kind = MP_TOKEN_NAME; From c889f01b8d72cb72dd4a8c627c43cd5f66c3d4d9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 12:29:51 +1100 Subject: [PATCH 83/92] tests/cmdline/cmd_parsetree: Update to work with changes to grammar. --- tests/cmdline/cmd_parsetree.py.exp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index 17e4c9638c..36f1f1b271 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -1,15 +1,15 @@ ---------------- [ 4] rule(1) (n=8) - tok(5) + tok(4) [ 4] rule(22) (n=4) id(i) -[ 4] rule(46) (n=1) +[ 4] rule(45) (n=1) NULL [ 5] rule(8) (n=0) NULL [ 6] rule(5) (n=2) id(a) - tok(15) + tok(14) [ 7] rule(5) (n=2) id(b) str(str) From d87c6b676855a8f19586e7ac637fe5f1dbc31cbc Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 12:30:27 +1100 Subject: [PATCH 84/92] tests/basics/string_join: Add more tests for string concatenation. --- tests/basics/string_join.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/basics/string_join.py b/tests/basics/string_join.py index b8694c01e0..4a2e9aa913 100644 --- a/tests/basics/string_join.py +++ b/tests/basics/string_join.py @@ -25,3 +25,13 @@ except TypeError: # joined by the compiler print("a" "b") +print("a" '''b''') +print("a" # inline comment + "b") +print("a" \ + "b") + +# the following should not be joined by the compiler +x = 'a' +'b' +print(x) From 5124a940670dbb5c07f6681070b3d1580c71d697 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 12:44:24 +1100 Subject: [PATCH 85/92] py/lexer: Convert mp_uint_t to size_t where appropriate. --- py/lexer.c | 16 ++++++++-------- py/lexer.h | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/py/lexer.c b/py/lexer.c index 329875ab06..9dcdd19eb5 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -157,7 +157,7 @@ STATIC void next_char(mp_lexer_t *lex) { } } -STATIC void indent_push(mp_lexer_t *lex, mp_uint_t indent) { +STATIC void indent_push(mp_lexer_t *lex, size_t indent) { if (lex->num_indent_level >= lex->alloc_indent_level) { // TODO use m_renew_maybe and somehow indicate an error if it fails... probably by using MP_TOKEN_MEMORY_ERROR lex->indent_level = m_renew(uint16_t, lex->indent_level, lex->alloc_indent_level, lex->alloc_indent_level + MICROPY_ALLOC_LEXEL_INDENT_INC); @@ -166,7 +166,7 @@ STATIC void indent_push(mp_lexer_t *lex, mp_uint_t indent) { lex->indent_level[lex->num_indent_level++] = indent; } -STATIC mp_uint_t indent_top(mp_lexer_t *lex) { +STATIC size_t indent_top(mp_lexer_t *lex) { return lex->indent_level[lex->num_indent_level - 1]; } @@ -263,7 +263,7 @@ STATIC const char *const tok_kw[] = { // This is called with CUR_CHAR() before first hex digit, and should return with // it pointing to last hex digit // num_digits must be greater than zero -STATIC bool get_hex(mp_lexer_t *lex, mp_uint_t num_digits, mp_uint_t *result) { +STATIC bool get_hex(mp_lexer_t *lex, size_t num_digits, mp_uint_t *result) { mp_uint_t num = 0; while (num_digits-- != 0) { next_char(lex); @@ -354,7 +354,7 @@ STATIC void parse_string_literal(mp_lexer_t *lex, bool is_raw) { default: if (c >= '0' && c <= '7') { // Octal sequence, 1-3 chars - mp_uint_t digits = 3; + size_t digits = 3; mp_uint_t num = c - '0'; while (is_following_odigit(lex) && --digits != 0) { next_char(lex); @@ -458,7 +458,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { } else if (had_physical_newline && lex->nested_bracket_level == 0) { lex->tok_kind = MP_TOKEN_NEWLINE; - mp_uint_t num_spaces = lex->column - 1; + size_t num_spaces = lex->column - 1; if (num_spaces == indent_top(lex)) { } else if (num_spaces > indent_top(lex)) { indent_push(lex, num_spaces); @@ -622,7 +622,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { // search for encoded delimiter or operator const char *t = tok_enc; - mp_uint_t tok_enc_index = 0; + size_t tok_enc_index = 0; for (; *t != 0 && !is_char(lex, *t); t += 1) { if (*t == 'e' || *t == 'c') { t += 1; @@ -644,7 +644,7 @@ void mp_lexer_to_next(mp_lexer_t *lex) { // get the maximum characters for a valid token t += 1; - mp_uint_t t_index = tok_enc_index; + size_t t_index = tok_enc_index; for (;;) { for (; *t == 'e'; t += 1) { t += 1; @@ -762,7 +762,7 @@ mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader) { return lex; } -mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len) { +mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, size_t len, size_t free_len) { mp_reader_t reader; if (!mp_reader_new_mem(&reader, (const byte*)str, len, free_len)) { return NULL; diff --git a/py/lexer.h b/py/lexer.h index d5382fc6a3..5d998b3521 100644 --- a/py/lexer.h +++ b/py/lexer.h @@ -151,24 +151,24 @@ typedef struct _mp_lexer_t { unichar chr0, chr1, chr2; // current cached characters from source - mp_uint_t line; // current source line - mp_uint_t column; // current source column + size_t line; // current source line + size_t column; // current source column mp_int_t emit_dent; // non-zero when there are INDENT/DEDENT tokens to emit mp_int_t nested_bracket_level; // >0 when there are nested brackets over multiple lines - mp_uint_t alloc_indent_level; - mp_uint_t num_indent_level; + size_t alloc_indent_level; + size_t num_indent_level; uint16_t *indent_level; - mp_uint_t tok_line; // token source line - mp_uint_t tok_column; // token source column + size_t tok_line; // token source line + size_t tok_column; // token source column mp_token_kind_t tok_kind; // token kind vstr_t vstr; // token data } mp_lexer_t; mp_lexer_t *mp_lexer_new(qstr src_name, mp_reader_t reader); -mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len); +mp_lexer_t *mp_lexer_new_from_str_len(qstr src_name, const char *str, size_t len, size_t free_len); void mp_lexer_free(mp_lexer_t *lex); void mp_lexer_to_next(mp_lexer_t *lex); From bdebfaa4bf422ef164fa78257b59415e5dffc6cb Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 12:48:45 +1100 Subject: [PATCH 86/92] py/grammar: Remove unused rule. Since the recent changes to string/bytes literal concatenation, this rule is no longer used. --- py/grammar.h | 1 - 1 file changed, 1 deletion(-) diff --git a/py/grammar.h b/py/grammar.h index 7d75881e1c..e8041c0e08 100644 --- a/py/grammar.h +++ b/py/grammar.h @@ -269,7 +269,6 @@ DEF_RULE_NC(power_dbl_star, and_ident(2), tok(OP_DBL_STAR), rule(factor)) // trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME DEF_RULE_NC(atom, or(12), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), tok(STRING), tok(BYTES), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace)) -DEF_RULE_NC(string_or_bytes, or(2), tok(STRING), tok(BYTES)) DEF_RULE(atom_paren, c(atom_paren), and(3), tok(DEL_PAREN_OPEN), opt_rule(atom_2b), tok(DEL_PAREN_CLOSE)) DEF_RULE_NC(atom_2b, or(2), rule(yield_expr), rule(testlist_comp)) DEF_RULE(atom_bracket, c(atom_bracket), and(3), tok(DEL_BRACKET_OPEN), opt_rule(testlist_comp), tok(DEL_BRACKET_CLOSE)) From c9b0f0b248ffc95545abc2347106fc3684d3c002 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 13:07:42 +1100 Subject: [PATCH 87/92] stmhal/main: Remove unnecessary header includes. --- stmhal/main.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/stmhal/main.c b/stmhal/main.c index bcc429df2f..c6d482f3ce 100644 --- a/stmhal/main.c +++ b/stmhal/main.c @@ -27,15 +27,10 @@ #include #include -#include "py/nlr.h" -#include "py/lexer.h" -#include "py/parse.h" -#include "py/obj.h" #include "py/runtime.h" #include "py/stackctrl.h" #include "py/gc.h" #include "py/mphal.h" - #include "lib/utils/pyexec.h" #include "lib/oofatfs/ff.h" #include "extmod/vfs.h" From 9e2b2a1c177b8b5b611e9e471ba37f3a5e6136f3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 13:08:09 +1100 Subject: [PATCH 88/92] teensy/main: Remove unnecessary header includes. --- teensy/main.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/teensy/main.c b/teensy/main.c index ba7207fe83..d62ae3bdb7 100644 --- a/teensy/main.c +++ b/teensy/main.c @@ -3,8 +3,6 @@ #include #include -#include "py/nlr.h" -#include "py/parse.h" #include "py/lexer.h" #include "py/runtime.h" #include "py/stackctrl.h" From 7d02cc5ec40af1d5dc5419ee9e717faf2851c9fb Mon Sep 17 00:00:00 2001 From: stijn Date: Thu, 16 Feb 2017 11:48:01 +0100 Subject: [PATCH 89/92] windows/.gitignore: Ignore VC.db and VC.opendb files from VS2015 Since VS2015 update 2 .db files are used for storing browsing info, instead of .sdf files. If users don't specify a location for these files excplicitly they end up in the project directory so ignore them. --- windows/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/windows/.gitignore b/windows/.gitignore index d26ef4db7f..ec28212111 100644 --- a/windows/.gitignore +++ b/windows/.gitignore @@ -7,3 +7,4 @@ *.ilk *.filters /build/* +*.VC.*db From 8f3e07f17d545d78fa031c1eba8e907a7cd08808 Mon Sep 17 00:00:00 2001 From: Stephan Brauer Date: Thu, 16 Feb 2017 21:49:25 +0100 Subject: [PATCH 90/92] drivers/display/lcd160cr: Use correct variable in set_power(). --- drivers/display/lcd160cr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/display/lcd160cr.py b/drivers/display/lcd160cr.py index 0861f8233c..a5da643487 100644 --- a/drivers/display/lcd160cr.py +++ b/drivers/display/lcd160cr.py @@ -203,7 +203,7 @@ class LCD160CR: #### SETUP COMMANDS #### def set_power(self, on): - self.pwr(value) + self.pwr(on) sleep_ms(15) def set_orient(self, orient): From d80df91ef2818ccef3185c9ecffcc8c396cc7d9f Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 17 Feb 2017 16:57:22 +1100 Subject: [PATCH 91/92] docs/library/lcd160cr: Mention the valid values for set_power() method. --- docs/library/lcd160cr.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/library/lcd160cr.rst b/docs/library/lcd160cr.rst index 39f492fc41..56af097a17 100644 --- a/docs/library/lcd160cr.rst +++ b/docs/library/lcd160cr.rst @@ -103,7 +103,8 @@ Setup commands .. method:: LCD160CR.set_power(on) - Turn the display on or off, depending on the given value. + Turn the display on or off, depending on the given value of `on`: 0 or `False` + will turn the display off, and 1 or `True` will turn it on. .. method:: LCD160CR.set_orient(orient) From 3d739eb398e44c3151bb999fcb3427b5184b4f58 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 17 Feb 2017 22:06:52 +0300 Subject: [PATCH 92/92] zephyr/README: Network startup issues with frdm_k64f resolved. But leave a generic warning that users should be aware of Zephyr's limitations/issues for a board they use. --- zephyr/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zephyr/README.md b/zephyr/README.md index 65729df5b9..6e9e5d8492 100644 --- a/zephyr/README.md +++ b/zephyr/README.md @@ -59,10 +59,12 @@ it's a sign that you didn't followed instructions above. If you would like to just run it quickly without extra setup, see "minimal" build below. For deploying/flashing a firmware on a real board, follow Zephyr -documentation for a given board (mind again that networking is enabled -for the build, so you should be aware of known issues for a particular -board; for example, frdm_k64f requires Ethernet cable connected to both -board and host or it will hang/crash on startup). +documentation for a given board, including known issues for that board +(if any). (Mind again that networking is enabled for the default build, +so you should know if there're any special requirements in that regard, +cf. for example QEMU networking requirements above; real hardware boards +generally should not have any special requirements, unless there're known +issues). Quick example