From 3f8d34ca83c75283f9dca398158b58ba4706b58e Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 10 May 2014 23:03:30 +0300 Subject: [PATCH 01/10] objlist: Support list slice deletion. --- py/objlist.c | 18 ++++++++++++++++++ tests/basics/list_slice_assign.py | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/py/objlist.c b/py/objlist.c index 4432b2447b..9e30ebb4aa 100644 --- a/py/objlist.c +++ b/py/objlist.c @@ -153,6 +153,24 @@ STATIC mp_obj_t list_binary_op(int op, mp_obj_t lhs, mp_obj_t rhs) { STATIC mp_obj_t list_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { if (value == MP_OBJ_NULL) { // delete +#if MICROPY_ENABLE_SLICE + if (MP_OBJ_IS_TYPE(index, &mp_type_slice)) { + mp_obj_list_t *self = self_in; + machine_uint_t start, stop; + if (!mp_seq_get_fast_slice_indexes(self->len, index, &start, &stop)) { + assert(0); + } + + int len_adj = start - stop; + //printf("Len adj: %d\n", len_adj); + assert(len_adj <= 0); + mp_seq_replace_slice_no_grow(self->items, self->len, start, stop, self->items/*NULL*/, 0, mp_obj_t); + // Clear "freed" elements at the end of list + mp_seq_clear(self->items, self->len + len_adj, self->len, sizeof(*self->items)); + self->len += len_adj; + return mp_const_none; + } +#endif mp_obj_t args[2] = {self_in, index}; list_pop(2, args); return mp_const_none; diff --git a/tests/basics/list_slice_assign.py b/tests/basics/list_slice_assign.py index f880520461..baa9a00810 100644 --- a/tests/basics/list_slice_assign.py +++ b/tests/basics/list_slice_assign.py @@ -11,6 +11,9 @@ print(l) l = list(x) l[1:3] = [] print(l) +l = list(x) +del l[1:3] +print(l) l = list(x) l[:3] = [10, 20] @@ -18,6 +21,9 @@ print(l) l = list(x) l[:3] = [] print(l) +l = list(x) +del l[:3] +print(l) l = list(x) l[:-3] = [10, 20] @@ -25,3 +31,6 @@ print(l) l = list(x) l[:-3] = [] print(l) +l = list(x) +del l[:-3] +print(l) From 7aca1cae340835be70060822bea1d0d8bc0082e8 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 02:26:42 +0300 Subject: [PATCH 02/10] py: Start making good use of mp_const_obj_t. --- py/obj.c | 6 +++--- py/obj.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/py/obj.c b/py/obj.c index 6494a847a5..e4bf7d7dfd 100644 --- a/py/obj.c +++ b/py/obj.c @@ -36,18 +36,18 @@ #include "runtime0.h" #include "runtime.h" -mp_obj_type_t *mp_obj_get_type(mp_obj_t o_in) { +mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in) { if (MP_OBJ_IS_SMALL_INT(o_in)) { return (mp_obj_t)&mp_type_int; } else if (MP_OBJ_IS_QSTR(o_in)) { return (mp_obj_t)&mp_type_str; } else { - mp_obj_base_t *o = o_in; + const mp_obj_base_t *o = o_in; return (mp_obj_t)o->type; } } -const char *mp_obj_get_type_str(mp_obj_t o_in) { +const char *mp_obj_get_type_str(mp_const_obj_t o_in) { return qstr_str(mp_obj_get_type(o_in)->name); } diff --git a/py/obj.h b/py/obj.h index 2418b28453..5757810c96 100644 --- a/py/obj.h +++ b/py/obj.h @@ -398,8 +398,8 @@ mp_obj_t mp_obj_new_bound_meth(mp_obj_t meth, mp_obj_t self); mp_obj_t mp_obj_new_getitem_iter(mp_obj_t *args); mp_obj_t mp_obj_new_module(qstr module_name); -mp_obj_type_t *mp_obj_get_type(mp_obj_t o_in); -const char *mp_obj_get_type_str(mp_obj_t o_in); +mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in); +const char *mp_obj_get_type_str(mp_const_obj_t o_in); bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo); // arguments should be type objects void mp_obj_print_helper(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t o_in, mp_print_kind_t kind); From 285683d2035d4a72169f6b90270a15e3eec38e7e Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 02:27:42 +0300 Subject: [PATCH 03/10] objboundmeth: If detailed reporting enabled, print object content. Similar to closure and cell. --- py/objboundmeth.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/py/objboundmeth.c b/py/objboundmeth.c index 85094add6a..9bbd34e9dd 100644 --- a/py/objboundmeth.c +++ b/py/objboundmeth.c @@ -39,6 +39,17 @@ typedef struct _mp_obj_bound_meth_t { mp_obj_t self; } mp_obj_bound_meth_t; +#if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_DETAILED +STATIC void bound_meth_print(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t o_in, mp_print_kind_t kind) { + mp_obj_bound_meth_t *o = o_in; + print(env, "self, PRINT_REPR); + print(env, "."); + mp_obj_print_helper(print, env, o->meth, PRINT_REPR); + print(env, ">"); +} +#endif + mp_obj_t bound_meth_call(mp_obj_t self_in, uint n_args, uint n_kw, const mp_obj_t *args) { mp_obj_bound_meth_t *self = self_in; @@ -65,6 +76,9 @@ mp_obj_t bound_meth_call(mp_obj_t self_in, uint n_args, uint n_kw, const mp_obj_ const mp_obj_type_t bound_meth_type = { { &mp_type_type }, .name = MP_QSTR_bound_method, +#if MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_DETAILED + .print = bound_meth_print, +#endif .call = bound_meth_call, }; From 69f3eb2c9688752d674254c3f4158cae688053be Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 02:44:46 +0300 Subject: [PATCH 04/10] objstr: Make .[r]partition() work with bytes. --- py/objstr.c | 22 +++++++++++++++++----- tests/basics/string_partition.py | 11 +++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index 80b0cf8766..247cfde6d5 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -54,6 +54,11 @@ STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str); STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str); STATIC mp_obj_t str_new(const mp_obj_type_t *type, const byte* data, uint len); STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in); +STATIC NORETURN void arg_type_mixup(); + +STATIC bool is_str_or_bytes(mp_obj_t o) { + return MP_OBJ_IS_STR(o) || MP_OBJ_IS_TYPE(o, &mp_type_bytes); +} /******************************************************************************/ /* str */ @@ -1326,9 +1331,12 @@ STATIC mp_obj_t str_count(uint n_args, const mp_obj_t *args) { } STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, machine_int_t direction) { - assert(MP_OBJ_IS_STR(self_in)); - if (!MP_OBJ_IS_STR(arg)) { - bad_implicit_conversion(arg); + if (!is_str_or_bytes(self_in)) { + assert(0); + } + mp_obj_type_t *self_type = mp_obj_get_type(self_in); + if (self_type != mp_obj_get_type(arg)) { + arg_type_mixup(); } GET_STR_DATA_LEN(self_in, str, str_len); @@ -1349,9 +1357,9 @@ STATIC mp_obj_t str_partitioner(mp_obj_t self_in, mp_obj_t arg, machine_int_t di const byte *position_ptr = find_subbytes(str, str_len, sep, sep_len, direction); if (position_ptr != NULL) { machine_uint_t position = position_ptr - str; - result[0] = mp_obj_new_str(str, position, false); + result[0] = str_new(self_type, str, position); result[1] = arg; - result[2] = mp_obj_new_str(str + position + sep_len, str_len - position - sep_len, false); + result[2] = str_new(self_type, str + position + sep_len, str_len - position - sep_len); } return mp_obj_new_tuple(3, result); @@ -1586,6 +1594,10 @@ STATIC void bad_implicit_conversion(mp_obj_t self_in) { nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "Can't convert '%s' object to str implicitly", mp_obj_get_type_str(self_in))); } +STATIC void arg_type_mixup() { + nlr_raise(mp_obj_new_exception_msg(&mp_type_TypeError, "Can't mix str and bytes arguments")); +} + uint mp_obj_str_get_hash(mp_obj_t self_in) { // TODO: This has too big overhead for hash accessor if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) { diff --git a/tests/basics/string_partition.py b/tests/basics/string_partition.py index ad70d02509..fe0070a658 100644 --- a/tests/basics/string_partition.py +++ b/tests/basics/string_partition.py @@ -27,3 +27,14 @@ except ValueError: print("Raised ValueError") else: print("Did not raise ValueError") + +# Bytes +print(b"abba".partition(b'b')) +try: + print(b"abba".partition('b')) +except TypeError: + print("Raised TypeError") +try: + print("abba".partition(b'b')) +except TypeError: + print("Raised TypeError") From 9511f60f016547ab00f634d451c230351bd8b225 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 03:12:36 +0300 Subject: [PATCH 05/10] py: Don't try to "bind" types store as attributes of objects. This was hit when trying to make urlparse.py from stdlib run. Took quite some time to debug. TODO: Reconsile bound method creation process better, maybe callable is to generic type to bind at all? --- py/objtype.c | 3 +++ py/runtime.c | 3 +++ tests/basics/class_store_class.py | 44 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 tests/basics/class_store_class.py diff --git a/py/objtype.c b/py/objtype.c index 345eee7140..ef5f6b9d9c 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -333,6 +333,9 @@ STATIC void instance_convert_return_attr(mp_obj_t self, mp_obj_t member, mp_obj_ // return a bound method, with self being the type of this object dest[0] = ((mp_obj_static_class_method_t*)member)->fun; dest[1] = mp_obj_get_type(self); + } else if (MP_OBJ_IS_TYPE(member, &mp_type_type)) { + // Don't try to bind types + dest[0] = member; } else if (mp_obj_is_callable(member)) { // return a bound method, with self being this object dest[0] = member; diff --git a/py/runtime.c b/py/runtime.c index 1479d8cba3..a2de2d75b1 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -840,6 +840,9 @@ void mp_load_method_maybe(mp_obj_t base, qstr attr, mp_obj_t *dest) { // return a bound method, with self being the type of this object dest[0] = ((mp_obj_static_class_method_t*)elem->value)->fun; dest[1] = mp_obj_get_type(base); + } else if (MP_OBJ_IS_TYPE(elem->value, &mp_type_type)) { + // Don't try to bind types + dest[0] = elem->value; } else if (mp_obj_is_callable(elem->value)) { // return a bound method, with self being this object dest[0] = elem->value; diff --git a/tests/basics/class_store_class.py b/tests/basics/class_store_class.py new file mode 100644 index 0000000000..c765fceaa8 --- /dev/null +++ b/tests/basics/class_store_class.py @@ -0,0 +1,44 @@ +# Inspired by urlparse.py from CPython 3.3 stdlib +# There was a bug in MicroPython that under some conditions class stored +# in instance attribute later was returned "bound" as if it was a method, +# which caused class constructor to receive extra argument. +from collections import namedtuple + +_DefragResultBase = namedtuple('DefragResult', 'foo bar') + +class _ResultMixinStr(object): + def encode(self): + return self._encoded_counterpart(*(x.encode() for x in self)) + +class _ResultMixinBytes(object): + def decode(self): + return self._decoded_counterpart(*(x.decode() for x in self)) + +class DefragResult(_DefragResultBase, _ResultMixinStr): + pass + +class DefragResultBytes(_DefragResultBase, _ResultMixinBytes): + pass + + +DefragResult._encoded_counterpart = DefragResultBytes +DefragResultBytes._decoded_counterpart = DefragResult + +# Due to differences in type and native subclass printing, +# the best thing we can do here is to just test that no exceptions +# happen + +#print(DefragResult, DefragResult._encoded_counterpart) +#print(DefragResultBytes, DefragResultBytes._decoded_counterpart) + +o1 = DefragResult("a", "b") +#print(o1, type(o1)) +o2 = DefragResultBytes("a", "b") +#print(o2, type(o2)) + +#print(o1._encoded_counterpart) +_o1 = o1.encode() +print(_o1[0], _o1[1]) +#print(_o1, type(_o1)) + +print("All's ok") From ea9708092e8a49377a465ef8c8500943fe9ba772 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 03:16:04 +0300 Subject: [PATCH 06/10] objtuple: Go out of the way to support comparison of subclasses. Two things are handled here: allow to compare native subtypes of tuple, e.g. namedtuple (TODO: should compare type too, currently compared duck-typedly by content). Secondly, allow user sunclasses of tuples (and its subtypes) be compared either. "Magic" I did previously in objtype.c covers only one argument (lhs is many), so we're in trouble when lhs is native type - there's no other option besides handling rhs in special manner. Fortunately, this patch outlines approach with fast path for native types. --- py/obj.h | 1 + py/objtuple.c | 14 +++++++++++--- py/objtype.c | 9 +++++++++ tests/basics/subclass_native_cmp.py | 9 +++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tests/basics/subclass_native_cmp.py diff --git a/py/obj.h b/py/obj.h index 5757810c96..30a60b77d0 100644 --- a/py/obj.h +++ b/py/obj.h @@ -401,6 +401,7 @@ mp_obj_t mp_obj_new_module(qstr module_name); mp_obj_type_t *mp_obj_get_type(mp_const_obj_t o_in); const char *mp_obj_get_type_str(mp_const_obj_t o_in); bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo); // arguments should be type objects +mp_obj_t mp_instance_cast_to_native_base(mp_const_obj_t self_in, mp_const_obj_t native_type); void mp_obj_print_helper(void (*print)(void *env, const char *fmt, ...), void *env, mp_obj_t o_in, mp_print_kind_t kind); void mp_obj_print(mp_obj_t o, mp_print_kind_t kind); diff --git a/py/objtuple.c b/py/objtuple.c index 7d4e87755f..ca65b28e31 100644 --- a/py/objtuple.c +++ b/py/objtuple.c @@ -99,12 +99,20 @@ mp_obj_t mp_obj_tuple_make_new(mp_obj_t type_in, uint n_args, uint n_kw, const m // Don't pass MP_BINARY_OP_NOT_EQUAL here STATIC bool tuple_cmp_helper(int op, mp_obj_t self_in, mp_obj_t another_in) { - assert(MP_OBJ_IS_TYPE(self_in, &mp_type_tuple)); - if (!MP_OBJ_IS_TYPE(another_in, &mp_type_tuple)) { - return false; + mp_obj_type_t *self_type = mp_obj_get_type(self_in); + if (self_type->getiter != tuple_getiter) { + assert(0); } + mp_obj_type_t *another_type = mp_obj_get_type(another_in); mp_obj_tuple_t *self = self_in; mp_obj_tuple_t *another = another_in; + if (another_type->getiter != tuple_getiter) { + // Slow path for user subclasses + another = mp_instance_cast_to_native_base(another, &mp_type_tuple); + if (another == MP_OBJ_NULL) { + return false; + } + } return mp_seq_cmp_objs(op, self->items, self->len, another->items, another->len); } diff --git a/py/objtype.c b/py/objtype.c index ef5f6b9d9c..c579477db7 100644 --- a/py/objtype.c +++ b/py/objtype.c @@ -845,6 +845,15 @@ STATIC mp_obj_t mp_builtin_isinstance(mp_obj_t object, mp_obj_t classinfo) { MP_DEFINE_CONST_FUN_OBJ_2(mp_builtin_isinstance_obj, mp_builtin_isinstance); +mp_obj_t mp_instance_cast_to_native_base(mp_const_obj_t self_in, mp_const_obj_t native_type) { + mp_obj_type_t *self_type = mp_obj_get_type(self_in); + if (!mp_obj_is_subclass_fast(self_type, native_type)) { + return MP_OBJ_NULL; + } + mp_obj_instance_t *self = (mp_obj_instance_t*)self_in; + return self->subobj[0]; +} + /******************************************************************************/ // staticmethod and classmethod types (probably should go in a different file) diff --git a/tests/basics/subclass_native_cmp.py b/tests/basics/subclass_native_cmp.py new file mode 100644 index 0000000000..1a095bfa1a --- /dev/null +++ b/tests/basics/subclass_native_cmp.py @@ -0,0 +1,9 @@ +# Test calling non-special method inherited from native type + +class mytuple(tuple): + pass + +t = mytuple((1, 2, 3)) +print(t) +print(t == (1, 2, 3)) +print((1, 2, 3) == t) From b4acd028b613a721ffbe5a3136700f190635f7c9 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 03:40:32 +0300 Subject: [PATCH 07/10] tests: Fix import. --- tests/basics/class_store_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/basics/class_store_class.py b/tests/basics/class_store_class.py index c765fceaa8..cc80fb5b1c 100644 --- a/tests/basics/class_store_class.py +++ b/tests/basics/class_store_class.py @@ -2,7 +2,7 @@ # There was a bug in MicroPython that under some conditions class stored # in instance attribute later was returned "bound" as if it was a method, # which caused class constructor to receive extra argument. -from collections import namedtuple +from _collections import namedtuple _DefragResultBase = namedtuple('DefragResult', 'foo bar') From ce6c10172be6d17f76779ece930908f6248a1706 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 03:45:42 +0300 Subject: [PATCH 08/10] tests: Really fix import. --- tests/basics/class_store_class.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/basics/class_store_class.py b/tests/basics/class_store_class.py index cc80fb5b1c..60f65220d9 100644 --- a/tests/basics/class_store_class.py +++ b/tests/basics/class_store_class.py @@ -2,7 +2,10 @@ # There was a bug in MicroPython that under some conditions class stored # in instance attribute later was returned "bound" as if it was a method, # which caused class constructor to receive extra argument. -from _collections import namedtuple +try: + from collections import namedtuple +except ImportError: + from _collections import namedtuple _DefragResultBase = namedtuple('DefragResult', 'foo bar') From b2d4fc06fc95d8e96eabd6ef470f0e871275fb82 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 13:17:29 +0300 Subject: [PATCH 09/10] objstr: Make *strip() accept bytes. --- py/objstr.c | 9 ++++++--- tests/basics/string_strip.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/py/objstr.c b/py/objstr.c index 247cfde6d5..c44e9ebf16 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -540,7 +540,8 @@ enum { LSTRIP, RSTRIP, STRIP }; STATIC mp_obj_t str_uni_strip(int type, uint n_args, const mp_obj_t *args) { assert(1 <= n_args && n_args <= 2); - assert(MP_OBJ_IS_STR(args[0])); + assert(is_str_or_bytes(args[0])); + const mp_obj_type_t *self_type = mp_obj_get_type(args[0]); const byte *chars_to_del; uint chars_to_del_len; @@ -550,7 +551,9 @@ STATIC mp_obj_t str_uni_strip(int type, uint n_args, const mp_obj_t *args) { chars_to_del = whitespace; chars_to_del_len = sizeof(whitespace); } else { - assert(MP_OBJ_IS_STR(args[1])); + if (mp_obj_get_type(args[1]) != self_type) { + arg_type_mixup(); + } GET_STR_DATA_LEN(args[1], s, l); chars_to_del = s; chars_to_del_len = l; @@ -594,7 +597,7 @@ STATIC mp_obj_t str_uni_strip(int type, uint n_args, const mp_obj_t *args) { assert(last_good_char_pos >= first_good_char_pos); //+1 to accomodate the last character machine_uint_t stripped_len = last_good_char_pos - first_good_char_pos + 1; - return mp_obj_new_str(orig_str + first_good_char_pos, stripped_len, false); + return str_new(self_type, orig_str + first_good_char_pos, stripped_len); } STATIC mp_obj_t str_strip(uint n_args, const mp_obj_t *args) { diff --git a/tests/basics/string_strip.py b/tests/basics/string_strip.py index 8e03eff93a..4684c2a248 100644 --- a/tests/basics/string_strip.py +++ b/tests/basics/string_strip.py @@ -10,3 +10,13 @@ print('www.example.com'.lstrip('cmowz.')) print(' spacious '.rstrip()) print('mississippi'.rstrip('ipz')) + +print(b'mississippi'.rstrip(b'ipz')) +try: + print(b'mississippi'.rstrip('ipz')) +except TypeError: + print("TypeError") +try: + print('mississippi'.rstrip(b'ipz')) +except TypeError: + print("TypeError") From eea01186547f0f1568ea1c8f002da4e33b7b0e46 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sun, 11 May 2014 13:51:24 +0300 Subject: [PATCH 10/10] py: Give up and make mp_obj_str_get_data() deal with bytes too. This is not fully correct re: error handling, because we should check that that types are used consistently (only str's or only bytes), but magically makes lot of functions support bytes. --- py/objstr.c | 2 +- tests/basics/int1.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/py/objstr.c b/py/objstr.c index c44e9ebf16..33bfcc3756 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -1647,7 +1647,7 @@ const char *mp_obj_str_get_str(mp_obj_t self_in) { } const char *mp_obj_str_get_data(mp_obj_t self_in, uint *len) { - if (MP_OBJ_IS_STR(self_in)) { + if (is_str_or_bytes(self_in)) { GET_STR_DATA_LEN(self_in, s, l); *len = l; return (const char*)s; diff --git a/tests/basics/int1.py b/tests/basics/int1.py index 2daef9bf0e..e8a0a04683 100644 --- a/tests/basics/int1.py +++ b/tests/basics/int1.py @@ -46,6 +46,7 @@ print(int('0B100', 2)) print(int('0100', 2)) print(int(' \t 0o12', 8)) print(int('0o12 \t ', 8)) +print(int(b"12", 10)) def test(value, base):