From 1022f9cc35564b216a4bcd7c65e8243c810a0ca9 Mon Sep 17 00:00:00 2001 From: Tom McDermott Date: Mon, 5 Aug 2019 15:15:28 +1000 Subject: [PATCH] py/modstruct: Fix struct.unpack with unaligned offset of native type. With this patch alignment is done relative to the start of the buffer that is being unpacked, not the raw pointer value, as per CPython. Fixes issue #3314. --- extmod/moductypes.c | 2 +- py/binary.c | 6 +++--- py/binary.h | 2 +- py/modstruct.c | 3 ++- tests/basics/struct_endian.py | 17 +++++++++++++++++ 5 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 tests/basics/struct_endian.py diff --git a/extmod/moductypes.c b/extmod/moductypes.c index 9b46371f36..d4c7611dfe 100644 --- a/extmod/moductypes.c +++ b/extmod/moductypes.c @@ -299,7 +299,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(uctypes_struct_sizeof_obj, 1, 2, ucty static inline mp_obj_t get_unaligned(uint val_type, byte *p, int big_endian) { char struct_type = big_endian ? '>' : '<'; static const char type2char[16] = "BbHhIiQq------fd"; - return mp_binary_get_val(struct_type, type2char[val_type], &p); + return mp_binary_get_val(struct_type, type2char[val_type], p, &p); } static inline void set_unaligned(uint val_type, byte *p, int big_endian, mp_obj_t val) { diff --git a/py/binary.c b/py/binary.c index a142776c37..9810e46600 100644 --- a/py/binary.c +++ b/py/binary.c @@ -185,14 +185,14 @@ long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, con } #define is_signed(typecode) (typecode > 'Z') -mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { +mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr) { byte *p = *ptr; mp_uint_t align; size_t size = mp_binary_get_size(struct_type, val_type, &align); if (struct_type == '@') { - // Make pointer aligned - p = (byte*)MP_ALIGN(p, (size_t)align); + // Align p relative to p_base + p = p_base + (uintptr_t)MP_ALIGN(p - p_base, (size_t)align); #if MP_ENDIANNESS_LITTLE struct_type = '<'; #else diff --git a/py/binary.h b/py/binary.h index 71182042f9..092b722886 100644 --- a/py/binary.h +++ b/py/binary.h @@ -38,7 +38,7 @@ size_t mp_binary_get_size(char struct_type, char val_type, mp_uint_t *palign); mp_obj_t mp_binary_get_val_array(char typecode, void *p, mp_uint_t index); void mp_binary_set_val_array(char typecode, void *p, mp_uint_t index, mp_obj_t val_in); void mp_binary_set_val_array_from_int(char typecode, void *p, mp_uint_t index, mp_int_t val); -mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr); +mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte *p_base, byte **ptr); void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **ptr); long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src); void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val); diff --git a/py/modstruct.c b/py/modstruct.c index 8617a8e0d3..957e4917b6 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -146,6 +146,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { } p += offset; } + byte *p_base = p; // Check that the input buffer is big enough to unpack all the values if (p + total_sz > end_p) { @@ -164,7 +165,7 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { res->items[i++] = item; } else { while (cnt--) { - item = mp_binary_get_val(fmt_type, *fmt, &p); + item = mp_binary_get_val(fmt_type, *fmt, p_base, &p); res->items[i++] = item; } } diff --git a/tests/basics/struct_endian.py b/tests/basics/struct_endian.py new file mode 100644 index 0000000000..ae32438245 --- /dev/null +++ b/tests/basics/struct_endian.py @@ -0,0 +1,17 @@ +# test ustruct and endian specific things + +try: + import ustruct as struct +except: + try: + import struct + except ImportError: + print("SKIP") + raise SystemExit + +# unpack/unpack_from with unaligned native type +buf = b'0123456789' +print(struct.unpack('h', memoryview(buf)[1:3])) +print(struct.unpack_from('i', buf, 1)) +print(struct.unpack_from('@i', buf, 1)) +print(struct.unpack_from('@ii', buf, 1))