From 752d2080b7af5b2bdd1328d17c87ccc5501c1e6a Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Fri, 18 Apr 2014 23:57:25 +0300 Subject: [PATCH 1/7] modffi: Mark 'p' type spec deprecated, replace with 'P'. 'p' in struct module is "pascal string". 'P' is void*. --- unix/modffi.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unix/modffi.c b/unix/modffi.c index 7b8e43874b..bdaa3b2b43 100644 --- a/unix/modffi.c +++ b/unix/modffi.c @@ -63,7 +63,8 @@ STATIC ffi_type *char2ffi_type(char c) case 'L': return &ffi_type_ulong; case 'f': return &ffi_type_float; case 'd': return &ffi_type_double; - case 'p': + case 'p': // Deprecated - conflicts with struct module + case 'P': case 's': return &ffi_type_pointer; case 'v': return &ffi_type_void; default: return NULL; From 5695e07256413fab8b280a939c264b06f7f5793f Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 19 Apr 2014 00:21:22 +0300 Subject: [PATCH 2/7] modstruct: Support 'q' & 'Q' type codes. --- py/binary.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/py/binary.c b/py/binary.c index d3dd009546..0c9db9bd80 100644 --- a/py/binary.c +++ b/py/binary.c @@ -92,6 +92,8 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { size = 4; break; case 'l': case 'L': size = 4; break; + case 'q': case 'Q': + size = 8; break; } break; case '@': { @@ -112,6 +114,9 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { align = size = sizeof(int); break; case 'l': case 'L': align = size = sizeof(long); break; + case 'q': case 'Q': + // TODO: This is for x86 + align = sizeof(int); size = sizeof(long long); break; } // Make pointer aligned p = (byte*)(((machine_uint_t)p + align - 1) & ~(align - 1)); From 1355cf42f23a39d3b887d2771d8bc7f3669d211c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 19 Apr 2014 01:25:49 +0300 Subject: [PATCH 3/7] modstruct: Fix .calcsize() to account for struct type/alignment. --- py/binary.c | 131 +++++++++++++++++++++---------------------------- py/binary.h | 2 +- py/modstruct.c | 6 ++- py/objarray.c | 6 +-- 4 files changed, 65 insertions(+), 80 deletions(-) diff --git a/py/binary.c b/py/binary.c index 0c9db9bd80..702a9cceb3 100644 --- a/py/binary.c +++ b/py/binary.c @@ -1,4 +1,5 @@ #include +#include #include #include "misc.h" @@ -9,34 +10,52 @@ // Helpers to work with binary-encoded data -int mp_binary_get_size(char typecode) { - // This assumes that unsigned and signed types are of the same type, - // which is invariant for [u]intN_t. - switch (typecode) { - case BYTEARRAY_TYPECODE: - case 'b': - case 'B': - return sizeof(int8_t); - case 'h': - case 'H': - return sizeof(int16_t); - case 'i': - case 'I': - return sizeof(int32_t); - case 'l': - case 'L': - return sizeof(int32_t); - case 'q': - case 'Q': - return sizeof(long long); -#if MICROPY_ENABLE_FLOAT - case 'f': - return sizeof(float); - case 'd': - return sizeof(double); -#endif +int mp_binary_get_size(char struct_type, char val_type, uint *palign) { + int size = 0; + int align = 1; + switch (struct_type) { + case '<': case '>': + switch (val_type) { + case 'b': case 'B': + size = 1; break; + case 'h': case 'H': + size = 2; break; + case 'i': case 'I': + size = 4; break; + case 'l': case 'L': + size = 4; break; + case 'q': case 'Q': + size = 8; break; + } + break; + case '@': { + // TODO: + // The simplest heuristic for alignment is to align by value + // size, but that doesn't work for "bigger than int" types, + // for example, long long may very well have long alignment + // So, we introduce separate alignment handling, but having + // formal support for that is different from actually supporting + // particular (or any) ABI. + switch (val_type) { + case BYTEARRAY_TYPECODE: + case 'b': case 'B': + align = size = 1; break; + case 'h': case 'H': + align = size = sizeof(short); break; + case 'i': case 'I': + align = size = sizeof(int); break; + case 'l': case 'L': + align = size = sizeof(long); break; + case 'q': case 'Q': + // TODO: This is for x86 + align = sizeof(int); size = sizeof(long long); break; + } + } } - return -1; + if (palign != NULL) { + *palign = align; + } + return size; } mp_obj_t mp_binary_get_val_array(char typecode, void *p, int index) { @@ -80,53 +99,17 @@ mp_obj_t mp_binary_get_val_array(char typecode, void *p, int index) { #define is_signed(typecode) (typecode > 'Z') mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { byte *p = *ptr; - uint size = 0; - switch (struct_type) { - case '<': case '>': - switch (val_type) { - case 'b': case 'B': - size = 1; break; - case 'h': case 'H': - size = 2; break; - case 'i': case 'I': - size = 4; break; - case 'l': case 'L': - size = 4; break; - case 'q': case 'Q': - size = 8; break; - } - break; - case '@': { - // TODO: - // The simplest heuristic for alignment is to align by value - // size, but that doesn't work for "bigger than int" types, - // for example, long long may very well have long alignment - // So, we introduce separate alignment handling, but having - // formal support for that is different from actually supporting - // particular (or any) ABI. - uint align = 0; - switch (val_type) { - case 'b': case 'B': - align = size = 1; break; - case 'h': case 'H': - align = size = sizeof(short); break; - case 'i': case 'I': - align = size = sizeof(int); break; - case 'l': case 'L': - align = size = sizeof(long); break; - case 'q': case 'Q': - // TODO: This is for x86 - align = sizeof(int); size = sizeof(long long); break; - } - // Make pointer aligned - p = (byte*)(((machine_uint_t)p + align - 1) & ~(align - 1)); - #if MP_ENDIANNESS_LITTLE - struct_type = '<'; - #else - struct_type = '>'; - #endif - break; - } + uint align; + + int size = mp_binary_get_size(struct_type, val_type, &align); + if (struct_type == '@') { + // Make pointer aligned + p = (byte*)(((machine_uint_t)p + align - 1) & ~(align - 1)); + #if MP_ENDIANNESS_LITTLE + struct_type = '<'; + #else + struct_type = '>'; + #endif } int delta; diff --git a/py/binary.h b/py/binary.h index 538d6e7f29..54ff2dc56b 100644 --- a/py/binary.h +++ b/py/binary.h @@ -2,7 +2,7 @@ // (underlyingly they're same). #define BYTEARRAY_TYPECODE 0 -int mp_binary_get_size(char typecode); +int mp_binary_get_size(char struct_type, char val_type, uint *palign); mp_obj_t mp_binary_get_val_array(char typecode, void *p, int index); mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr); void mp_binary_set_val_array(char typecode, void *p, int index, mp_obj_t val_in); diff --git a/py/modstruct.c b/py/modstruct.c index cd2516b240..0be194feec 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -37,12 +37,14 @@ STATIC uint calcsize_items(const char *fmt) { STATIC mp_obj_t struct_calcsize(mp_obj_t fmt_in) { const char *fmt = mp_obj_str_get_str(fmt_in); char fmt_type = get_fmt_type(&fmt); - (void)fmt_type; machine_uint_t size; for (size = 0; *fmt; fmt++) { - int sz = mp_binary_get_size(*fmt); + uint align; + int sz = mp_binary_get_size(fmt_type, *fmt, &align); // TODO assert(sz != -1); + // Apply alignment + size = (size + align - 1) & ~(align - 1); size += sz; } return MP_OBJ_NEW_SMALL_INT(size); diff --git a/py/objarray.c b/py/objarray.c index c6da45728a..2255e29d7b 100644 --- a/py/objarray.c +++ b/py/objarray.c @@ -121,7 +121,7 @@ STATIC mp_obj_t array_append(mp_obj_t self_in, mp_obj_t arg) { assert(MP_OBJ_IS_TYPE(self_in, &mp_type_array) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytearray)); mp_obj_array_t *self = self_in; if (self->free == 0) { - int item_sz = mp_binary_get_size(self->typecode); + int item_sz = mp_binary_get_size('@', self->typecode, NULL); // TODO: alloc policy self->free = 8; self->items = m_realloc(self->items, item_sz * self->len, item_sz * (self->len + self->free)); @@ -154,7 +154,7 @@ STATIC mp_obj_t array_subscr(mp_obj_t self_in, mp_obj_t index_in, mp_obj_t value STATIC machine_int_t array_get_buffer(mp_obj_t o_in, mp_buffer_info_t *bufinfo, int flags) { mp_obj_array_t *o = o_in; bufinfo->buf = o->items; - bufinfo->len = o->len * mp_binary_get_size(o->typecode); + bufinfo->len = o->len * mp_binary_get_size('@', o->typecode, NULL); bufinfo->typecode = o->typecode; return 0; } @@ -190,7 +190,7 @@ const mp_obj_type_t mp_type_bytearray = { }; STATIC mp_obj_array_t *array_new(char typecode, uint n) { - int typecode_size = mp_binary_get_size(typecode); + int typecode_size = mp_binary_get_size('@', typecode, NULL); if (typecode_size <= 0) { nlr_raise(mp_obj_new_exception_msg(&mp_type_ValueError, "bad typecode")); } From 4602b9a79f86176d31bf8d3760abf2d39a380b39 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 19 Apr 2014 03:07:34 +0300 Subject: [PATCH 4/7] obj.h: Typo fix in comment. --- py/obj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/obj.h b/py/obj.h index dd25ec4f22..44d9bf6a6d 100644 --- a/py/obj.h +++ b/py/obj.h @@ -418,7 +418,7 @@ machine_int_t mp_obj_int_get(mp_obj_t self_in); #if MICROPY_ENABLE_FLOAT mp_float_t mp_obj_int_as_float(mp_obj_t self_in); #endif -// Will rains exception if value doesn't fit into machine_int_t +// Will raise exception if value doesn't fit into machine_int_t machine_int_t mp_obj_int_get_checked(mp_obj_t self_in); // exception From 504e23388ce68fcb1ac80a7c411979718686af7c Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 19 Apr 2014 03:09:17 +0300 Subject: [PATCH 5/7] objstr: Init hash in mp_obj_str_builder_start() to 0. --- py/objstr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/py/objstr.c b/py/objstr.c index a682144b8f..b0d5cba6f1 100644 --- a/py/objstr.c +++ b/py/objstr.c @@ -1415,6 +1415,7 @@ mp_obj_t mp_obj_str_builder_start(const mp_obj_type_t *type, uint len, byte **da mp_obj_str_t *o = m_new_obj(mp_obj_str_t); o->base.type = type; o->len = len; + o->hash = 0; byte *p = m_new(byte, len + 1); o->data = p; *data = p; From 6204460461cc21b27861fa89b2423119f8cdce88 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 19 Apr 2014 03:13:15 +0300 Subject: [PATCH 6/7] modstruct: Initial implementation of struct.pack(). --- py/binary.c | 39 +++++++++++++++++++++++++++++++++++++++ py/binary.h | 3 ++- py/modstruct.c | 18 ++++++++++++++++++ py/qstrdefs.h | 12 ++++++------ tests/basics/struct1.py | 10 ++++++++++ 5 files changed, 75 insertions(+), 7 deletions(-) diff --git a/py/binary.c b/py/binary.c index 702a9cceb3..ee95d56e44 100644 --- a/py/binary.c +++ b/py/binary.c @@ -138,6 +138,45 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { } } +void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr) { + byte *p = *ptr; + uint align; + + int size = mp_binary_get_size(struct_type, val_type, &align); + if (struct_type == '@') { + // Make pointer aligned + p = (byte*)(((machine_uint_t)p + align - 1) & ~(align - 1)); + #if MP_ENDIANNESS_LITTLE + struct_type = '<'; + #else + struct_type = '>'; + #endif + } + +#if MP_ENDIANNESS_BIG +#error Not implemented +#endif + machine_int_t val = mp_obj_int_get_checked(val_in); + byte *in = (byte*)&val; + int in_delta, out_delta; + uint val_sz = MIN(size, sizeof(val)); + if (struct_type == '>') { + in_delta = -1; + out_delta = 1; + in += val_sz - 1; + } else { + in_delta = out_delta = 1; + } + + for (uint i = val_sz; i > 0; i--) { + *p = *in; + p += out_delta; + in += in_delta; + } + + *ptr += size; +} + void mp_binary_set_val_array(char typecode, void *p, int index, mp_obj_t val_in) { switch (typecode) { #if MICROPY_ENABLE_FLOAT diff --git a/py/binary.h b/py/binary.h index 54ff2dc56b..46fb5b36b9 100644 --- a/py/binary.h +++ b/py/binary.h @@ -4,6 +4,7 @@ int mp_binary_get_size(char struct_type, char val_type, uint *palign); mp_obj_t mp_binary_get_val_array(char typecode, void *p, int index); -mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr); void mp_binary_set_val_array(char typecode, void *p, int index, mp_obj_t val_in); void mp_binary_set_val_array_from_int(char typecode, void *p, int index, machine_int_t val); +mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr); +void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr); diff --git a/py/modstruct.c b/py/modstruct.c index 0be194feec..81afd94d16 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -6,6 +6,7 @@ #include "obj.h" #include "builtin.h" #include "objtuple.h" +#include "objstr.h" #include "binary.h" #if MICROPY_ENABLE_MOD_STRUCT @@ -69,9 +70,26 @@ STATIC mp_obj_t struct_unpack(mp_obj_t fmt_in, mp_obj_t data_in) { } MP_DEFINE_CONST_FUN_OBJ_2(struct_unpack_obj, struct_unpack); +STATIC mp_obj_t struct_pack(uint n_args, mp_obj_t *args) { + // TODO: "The arguments must match the values required by the format exactly." + const char *fmt = mp_obj_str_get_str(args[0]); + char fmt_type = get_fmt_type(&fmt); + int size = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0])); + byte *p; + mp_obj_t res = mp_obj_str_builder_start(&mp_type_bytes, size, &p); + memset(p, 0, size); + + for (uint i = 1; i < n_args; i++) { + mp_binary_set_val(fmt_type, *fmt++, args[i], &p); + } + return res; +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_obj, 1, -1, struct_pack); + STATIC const mp_map_elem_t mp_module_struct_globals_table[] = { { MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR_struct) }, { MP_OBJ_NEW_QSTR(MP_QSTR_calcsize), (mp_obj_t)&struct_calcsize_obj }, + { MP_OBJ_NEW_QSTR(MP_QSTR_pack), (mp_obj_t)&struct_pack_obj }, { MP_OBJ_NEW_QSTR(MP_QSTR_unpack), (mp_obj_t)&struct_unpack_obj }, }; diff --git a/py/qstrdefs.h b/py/qstrdefs.h index a4df86fda7..86a5f2632c 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -139,16 +139,10 @@ Q(staticmethod) Q(sum) Q(super) Q(str) -#if MICROPY_ENABLE_MOD_STRUCT -Q(struct) -#endif Q(sys) Q(to_bytes) Q(tuple) Q(type) -#if MICROPY_ENABLE_MOD_STRUCT -Q(unpack) -#endif Q(value) Q(zip) @@ -299,6 +293,12 @@ Q(version) Q(version_info) #endif +#if MICROPY_ENABLE_MOD_STRUCT +Q(struct) +Q(pack) +Q(unpack) +#endif + #if MICROPY_ENABLE_PROPERTY Q(property) Q(getter) diff --git a/tests/basics/struct1.py b/tests/basics/struct1.py index a32979bff9..3a05c85f0b 100644 --- a/tests/basics/struct1.py +++ b/tests/basics/struct1.py @@ -6,3 +6,13 @@ print(struct.unpack(">bI", b"\x80\0\0\x01\0")) # 32-bit little-endian specific #print(struct.unpack("bI", b"\x80\xaa\x55\xaa\0\0\x01\0")) + +print(struct.pack("i", 1)) +print(struct.pack("h", 1)) +print(struct.pack("b", 1)) + +print(struct.pack("bI", -128, 256)) From 206dd2a905fc4bca89c0b1a8beb2c71516f2053f Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 19 Apr 2014 03:27:37 +0300 Subject: [PATCH 7/7] stmhal: Update for mp_binary_get_size() refactor. --- stmhal/adc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stmhal/adc.c b/stmhal/adc.c index be83b03ad7..f03017ab4f 100644 --- a/stmhal/adc.c +++ b/stmhal/adc.c @@ -169,7 +169,7 @@ STATIC mp_obj_t adc_read_timed(mp_obj_t self_in, mp_obj_t buf_in, mp_obj_t freq_ mp_buffer_info_t bufinfo; mp_get_buffer_raise(buf_in, &bufinfo, MP_BUFFER_WRITE); - int typesize = mp_binary_get_size(bufinfo.typecode); + int typesize = mp_binary_get_size('@', bufinfo.typecode, NULL); // Init TIM6 at the required frequency (in Hz) timer_tim6_init(mp_obj_get_int(freq_in));