CircuitPython now has struct instead of ustruct (#302)

Added struct module to shared-bindings and shared-module.  removed ustruct

atmel-samd/Makefile

@@ -292,6 +292,7 @@ SRC_SHARED_MODULE = \
 	os/__init__.c \
 	random/__init__.c \
 	storage/__init__.c \
+	struct/__init__.c \
 	uheap/__init__.c \
 	ustack/__init__.c
 

atmel-samd/mpconfigport.h

@@ -56,7 +56,7 @@
 #define MICROPY_PY_IO               (0)
 #define MICROPY_PY_URANDOM          (0)
 #define MICROPY_PY_URANDOM_EXTRA_FUNCS (0)
-#define MICROPY_PY_STRUCT           (1)
+#define MICROPY_PY_STRUCT           (0)
 #define MICROPY_PY_SYS              (1)
 // If you change MICROPY_LONGINT_IMPL, also change MPY_TOOL_LONGINT_IMPL in mpconfigport.mk.
 #define MICROPY_LONGINT_IMPL        (MICROPY_LONGINT_IMPL_NONE)
@@ -161,6 +161,7 @@ extern const struct _mp_obj_module_t board_module;
 extern const struct _mp_obj_module_t os_module;
 extern const struct _mp_obj_module_t random_module;
 extern const struct _mp_obj_module_t storage_module;
+extern const struct _mp_obj_module_t struct_module;
 extern const struct _mp_obj_module_t time_module;
 extern const struct _mp_obj_module_t cpy_nvm_module;
 extern const struct _mp_obj_module_t neopixel_write_module;
@@ -223,6 +224,7 @@ extern const struct _mp_obj_module_t usb_hid_module;
     { MP_OBJ_NEW_QSTR(MP_QSTR_microcontroller), (mp_obj_t)&microcontroller_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_os), (mp_obj_t)&os_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_random), (mp_obj_t)&random_module }, \
+    { MP_OBJ_NEW_QSTR(MP_QSTR_struct), (mp_obj_t)&struct_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_time), (mp_obj_t)&time_module },
     EXTRA_BUILTIN_MODULES
 

esp8266/Makefile

@@ -143,7 +143,8 @@ SRC_SHARED_MODULE = \
 	multiterminal/__init__.c \
 	os/__init__.c \
 	random/__init__.c \
-	storage/__init__.c
+	storage/__init__.c \
+	struct/__init__.c 
 
 SRC_SHARED_MODULE_EXPANDED = $(addprefix shared-bindings/, $(SRC_SHARED_MODULE)) \
                              $(addprefix shared-module/, $(SRC_SHARED_MODULE))

esp8266/mpconfigport.h

@@ -55,7 +55,7 @@
 #define MICROPY_PY_CMATH            (0)
 #define MICROPY_PY_IO               (1)
 #define MICROPY_PY_IO_FILEIO        (1)
-#define MICROPY_PY_STRUCT           (1)
+#define MICROPY_PY_STRUCT           (0)
 #define MICROPY_PY_SYS              (1)
 #define MICROPY_PY_SYS_MAXSIZE      (1)
 #define MICROPY_PY_SYS_EXIT         (1)
@@ -159,6 +159,7 @@ extern const struct _mp_obj_module_t network_module;
 extern const struct _mp_obj_module_t os_module;
 extern const struct _mp_obj_module_t random_module;
 extern const struct _mp_obj_module_t storage_module;
+extern const struct _mp_obj_module_t struct_module;
 extern const struct _mp_obj_module_t mp_module_lwip;
 extern const struct _mp_obj_module_t mp_module_machine;
 extern const struct _mp_obj_module_t mp_module_onewire;
@@ -189,6 +190,7 @@ extern const struct _mp_obj_module_t multiterminal_module;
     { MP_OBJ_NEW_QSTR(MP_QSTR_bitbangio), (mp_obj_t)&bitbangio_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_storage), (mp_obj_t)&storage_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_random), (mp_obj_t)&random_module }, \
+    { MP_OBJ_NEW_QSTR(MP_QSTR_struct), (mp_obj_t)&struct_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_time), (mp_obj_t)&time_module }, \
     { MP_OBJ_NEW_QSTR(MP_QSTR_multiterminal), (mp_obj_t)&multiterminal_module }, \
 

shared-bindings/index.rst

@@ -30,6 +30,7 @@ Module / Port      SAMD21   SAMD21 Express  ESP8266
 `pulseio`          No       **Yes**         No
 `random`           **Yes**  **Yes**         **Yes**
 `storage`          **Yes**  **Yes**         **Yes**
+`struct`           **Yes**  **Yes**         **Yes**
 `time`             **Yes**  **Yes**         **Yes**
 `touchio`          **Yes**  **Yes**         No
 `uheap`            Debug    Debug           Debug

shared-bindings/struct/__init__.c

@@ -0,0 +1,168 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2013, 2014 Damien P. George
+ * Copyright (c) 2014 Paul Sokolovsky
+ * Copyright (c) 2017 Michael McWethy
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <assert.h>
+#include <string.h>
+
+#include "py/runtime.h"
+#include "py/builtin.h"
+#include "py/objtuple.h"
+#include "py/binary.h"
+#include "py/parsenum.h"
+#include "shared-bindings/struct/__init__.h"
+#include "shared-module/struct/__init__.h"
+
+//| :mod:`struct` --- manipulation of c-style data
+//| ========================================================
+//|
+//| .. module:: struct
+//|   :synopsis: byte data control
+//|   :platform: SAMD21
+//|
+//| This module implements a subset of the corresponding CPython module,
+//| as described below. For more information, refer to the original CPython
+//| documentation: struct.
+//|
+//| Supported size/byte order prefixes: *@*, *<*, *>*, *!*.
+//|
+//| Supported format codes: *b*, *B*, *h*, *H*, *i*, *I*, *l*, *L*, *q*, *Q*,
+//| *s*, *P*, *f*, *d* (the latter 2 depending on the floating-point support).
+
+
+ //| .. function:: calcsize(fmt)
+ //|
+ //|   Return the number of bytes needed to store the given fmt.
+ //|
+
+STATIC mp_obj_t struct_calcsize(mp_obj_t fmt_in) {
+
+    return MP_OBJ_NEW_SMALL_INT(shared_modules_struct_calcsize(fmt_in));
+}
+MP_DEFINE_CONST_FUN_OBJ_1(struct_calcsize_obj, struct_calcsize);
+
+//| .. function:: pack(fmt, v1, v2, ...)
+//|
+//|   Pack the values v1, v2, ... according to the format string fmt.
+//|   The return value is a bytes object encoding the values.
+//|
+
+STATIC mp_obj_t struct_pack(size_t n_args, const mp_obj_t *args) {
+    // TODO: "The arguments must match the values required by the format exactly."
+    mp_int_t size = MP_OBJ_SMALL_INT_VALUE(struct_calcsize(args[0]));
+    vstr_t vstr;
+    vstr_init_len(&vstr, size);
+    byte *p = (byte*)vstr.buf;
+    memset(p, 0, size);
+    byte *end_p = &p[size];
+    shared_modules_struct_pack_into(args[0], p, end_p, n_args - 1, &args[1]);
+    return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_obj, 1, MP_OBJ_FUN_ARGS_MAX, struct_pack);
+
+
+//| .. function:: pack_into(fmt, buffer, offset, v1, v2, ...)
+//|
+//|   Pack the values v1, v2, ... according to the format string fmt into a buffer
+//|   starting at offset. offset may be negative to count from the end of buffer.
+//|
+
+STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) {
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_WRITE);
+    mp_int_t offset = mp_obj_get_int(args[2]);
+    if (offset < 0) {
+        // negative offsets are relative to the end of the buffer
+        offset = (mp_int_t)bufinfo.len + offset;
+        if (offset < 0) {
+            mp_raise_RuntimeError("buffer too small");
+        }
+    }
+    byte *p = (byte *)bufinfo.buf;
+    byte *end_p = &p[bufinfo.len];
+    p += offset;
+
+    shared_modules_struct_pack_into(args[0], p, end_p, n_args - 3, &args[3]);
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_into_obj, 3, MP_OBJ_FUN_ARGS_MAX, struct_pack_into);
+
+//| .. function:: unpack(fmt, data)
+//|
+//|   Unpack from the data according to the format string fmt. The return value
+//|   is a tuple of the unpacked values.
+//|
+
+//| .. function:: unpack_from(fmt, data, offset)
+//|
+//|   Unpack from the data starting at offset according to the format string fmt.
+//|   offset may be negative to count from the end of buffer. The return value is
+//|   a tuple of the unpacked values.
+//|
+
+STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
+    // unpack requires that the buffer be exactly the right size.
+    // unpack_from requires that the buffer be "big enough".
+    // Since we implement unpack and unpack_from using the same function
+    // we relax the "exact" requirement, and only implement "big enough".
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ);
+    byte *p = bufinfo.buf;
+    byte *end_p = &p[bufinfo.len];
+
+    if (n_args > 2) {
+        mp_int_t offset = mp_obj_get_int(args[2]);
+        // offset arg provided
+        if (offset < 0) {
+            // negative offsets are relative to the end of the buffer
+            offset = bufinfo.len + offset;
+            if (offset < 0) {
+                mp_raise_RuntimeError("buffer too small");
+            }
+        }
+        p += offset;
+    }
+
+    return MP_OBJ_FROM_PTR(shared_modules_struct_unpack_from(args[0] , p, end_p));
+}
+MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_unpack_from_obj, 2, 3, struct_unpack_from);
+
+STATIC const mp_rom_map_elem_t mp_module_struct_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_struct) },
+    { MP_ROM_QSTR(MP_QSTR_calcsize), MP_ROM_PTR(&struct_calcsize_obj) },
+    { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&struct_pack_obj) },
+    { MP_ROM_QSTR(MP_QSTR_pack_into), MP_ROM_PTR(&struct_pack_into_obj) },
+    { MP_ROM_QSTR(MP_QSTR_unpack), MP_ROM_PTR(&struct_unpack_from_obj) },
+    { MP_ROM_QSTR(MP_QSTR_unpack_from), MP_ROM_PTR(&struct_unpack_from_obj) },
+};
+
+STATIC MP_DEFINE_CONST_DICT(mp_module_struct_globals, mp_module_struct_globals_table);
+
+const mp_obj_module_t struct_module = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t*)&mp_module_struct_globals,
+};

shared-bindings/struct/__init__.h

@@ -0,0 +1,34 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_STRUCT___INIT___H
+#define MICROPY_INCLUDED_SHARED_BINDINGS_STRUCT___INIT___H
+
+void shared_modules_struct_pack_into(mp_obj_t fmt_in, byte *p, byte* end_p, size_t n_args, const mp_obj_t *args);
+mp_uint_t shared_modules_struct_calcsize(mp_obj_t fmt_in);
+mp_obj_tuple_t * shared_modules_struct_unpack_from(mp_obj_t fmt_in, byte *p, byte *end_p);
+
+#endif  // MICROPY_INCLUDED_SHARED_BINDINGS_RANDOM___INIT___H

shared-module/struct/__init__.c

@@ -0,0 +1,190 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 Paul Sokolovsky
+ * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+ * Copyright (c) 2017 Michael McWethy
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <assert.h>
+#include <string.h>
+
+#include "py/runtime.h"
+#include "py/binary.h"
+#include "py/parsenum.h"
+
+void struct_validate_format(char fmt) {
+    if( fmt == 'S' || fmt == 'O') {
+        mp_raise_RuntimeError("'S' and 'O' are not supported format types");
+    }
+}
+
+char get_fmt_type(const char **fmt) {
+    char t = **fmt;
+    switch (t) {
+        case '!':
+            t = '>';
+            break;
+        case '@':
+        case '=':
+        case '<':
+        case '>':
+            break;
+        default:
+            return '@';
+    }
+    // Skip type char
+    (*fmt)++;
+    return t;
+}
+
+mp_uint_t get_fmt_num(const char **p) {
+    const char *num = *p;
+    uint len = 1;
+    while (unichar_isdigit(*++num)) {
+        len++;
+    }
+    mp_uint_t val = (mp_uint_t)MP_OBJ_SMALL_INT_VALUE(mp_parse_num_integer(*p, len, 10, NULL));
+    *p = num;
+    return val;
+}
+
+
+void shared_modules_struct_pack_into(mp_obj_t fmt_in, byte *p, byte* end_p, size_t n_args, const mp_obj_t *args) {
+    const char *fmt = mp_obj_str_get_str(fmt_in);
+    char fmt_type = get_fmt_type(&fmt);
+
+    size_t i;
+    for (i = 0; i < n_args;) {
+        mp_uint_t sz = 1;
+        if (*fmt == '\0') {
+            // more arguments given than used by format string; CPython raises struct.error here
+            mp_raise_RuntimeError("too many arguments provided with the given format");
+        }
+        struct_validate_format(*fmt);
+
+        if (unichar_isdigit(*fmt)) {
+            sz = get_fmt_num(&fmt);
+        }
+        if (p + sz > end_p) {
+            mp_raise_RuntimeError("buffer too small");
+        }
+
+        if (*fmt == 's') {
+            mp_buffer_info_t bufinfo;
+            mp_get_buffer_raise(args[i++], &bufinfo, MP_BUFFER_READ);
+            mp_uint_t to_copy = sz;
+            if (bufinfo.len < to_copy) {
+                to_copy = bufinfo.len;
+            }
+            memcpy(p, bufinfo.buf, to_copy);
+            memset(p + to_copy, 0, sz - to_copy);
+            p += sz;
+        } else {
+            while (sz--) {
+                mp_binary_set_val(fmt_type, *fmt, args[i++], &p);
+            }
+        }
+        fmt++;
+    }
+}
+
+mp_uint_t calcsize_items(const char *fmt) {
+    mp_uint_t cnt = 0;
+    while (*fmt) {
+        int num = 1;
+        if (unichar_isdigit(*fmt)) {
+            num = get_fmt_num(&fmt);
+            if (*fmt == 's') {
+                num = 1;
+            }
+        }
+        cnt += num;
+        fmt++;
+    }
+    return cnt;
+}
+
+mp_uint_t shared_modules_struct_calcsize(mp_obj_t fmt_in) {
+    const char *fmt = mp_obj_str_get_str(fmt_in);
+    char fmt_type = get_fmt_type(&fmt);
+
+    mp_uint_t size;
+    for (size = 0; *fmt; fmt++) {
+
+        struct_validate_format(*fmt);
+
+        mp_uint_t cnt = 1;
+        if (unichar_isdigit(*fmt)) {
+            cnt = get_fmt_num(&fmt);
+        }
+
+        if (*fmt == 's') {
+            size += cnt;
+        } else {
+            mp_uint_t align;
+            size_t sz = mp_binary_get_size(fmt_type, *fmt, &align);
+            while (cnt--) {
+                // Apply alignment
+                size = (size + align - 1) & ~(align - 1);
+                size += sz;
+            }
+        }
+    }
+    return size;
+}
+
+
+mp_obj_tuple_t * shared_modules_struct_unpack_from(mp_obj_t fmt_in, byte *p, byte *end_p) {
+
+  const char *fmt = mp_obj_str_get_str(fmt_in);
+  char fmt_type = get_fmt_type(&fmt);
+  mp_uint_t num_items = calcsize_items(fmt);
+  mp_obj_tuple_t *res = MP_OBJ_TO_PTR(mp_obj_new_tuple(num_items, NULL));
+
+  for (uint i = 0; i < num_items;) {
+      mp_uint_t sz = 1;
+
+      struct_validate_format(*fmt);
+
+      if (unichar_isdigit(*fmt)) {
+          sz = get_fmt_num(&fmt);
+      }
+      if (p + sz > end_p) {
+          mp_raise_RuntimeError("buffer too small");
+      }
+      mp_obj_t item;
+      if (*fmt == 's') {
+          item = mp_obj_new_bytes(p, sz);
+          p += sz;
+          res->items[i++] = item;
+      } else {
+          while (sz--) {
+              item = mp_binary_get_val(fmt_type, *fmt, &p);
+              res->items[i++] = item;
+          }
+      }
+      fmt++;
+  }
+  return res;
+
+}

shared-module/struct/__init__.h

@@ -0,0 +1,34 @@
+
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_SHARED_MODULE_STRUCT___INIT___H
+#define MICROPY_INCLUDED_SHARED_MODULE_STRUCT___INIT___H
+
+char get_fmt_type(const char **fmt);
+mp_uint_t get_fmt_num(const char **p);
+mp_uint_t calcsize_items(const char *fmt);
+
+#endif

tests/misc/non_compliant.py

@@ -2,7 +2,7 @@
 
 try:
     import array
-    import ustruct
+    import struct
 except ImportError:
     print("SKIP")
     raise SystemExit
@@ -118,10 +118,10 @@ except NotImplementedError:
     print('NotImplementedError')
 
 # struct pack with too many args, not checked by uPy
-print(ustruct.pack('bb', 1, 2, 3))
+print(struct.pack('bb', 1, 2, 3))
 
 # struct pack with too few args, not checked by uPy
-print(ustruct.pack('bb', 1))
+print(struct.pack('bb', 1))
 
 # array slice assignment with unsupported RHS
 try: