From 90c203a3dd0c25b39909e98ec64dcdd0fc73054c Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Sun, 8 Nov 2020 18:39:20 -0800
Subject: [PATCH 01/32] add module msgpack

---
 ports/nrf/mpconfigport.mk          |   2 +
 py/circuitpy_defns.mk              |   4 +
 py/circuitpy_mpconfig.h            |   8 +
 py/circuitpy_mpconfig.mk           |   4 +
 shared-bindings/msgpack/__init__.c | 391 +++++++++++++++++++++++++++++
 shared-bindings/msgpack/__init__.h |  34 +++
 6 files changed, 443 insertions(+)
 create mode 100644 shared-bindings/msgpack/__init__.c
 create mode 100644 shared-bindings/msgpack/__init__.h

diff --git a/ports/nrf/mpconfigport.mk b/ports/nrf/mpconfigport.mk
index 9560064fbc..2ee182b7d5 100644
--- a/ports/nrf/mpconfigport.mk
+++ b/ports/nrf/mpconfigport.mk
@@ -52,6 +52,8 @@ MCU_SUB_VARIANT = nrf52840
 # Fits on nrf52840 but space is tight on nrf52833.
 CIRCUITPY_AESIO ?= 1
 
+CIRCUITPY_MSGPACK ?= 1
+
 SD ?= s140
 SOFTDEV_VERSION ?= 6.1.0
 
diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk
index ccdf973e9f..144168c79e 100644
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@@ -283,6 +283,9 @@ endif
 ifeq ($(CIRCUITPY_PEW),1)
 SRC_PATTERNS += _pew/%
 endif
+ifeq ($(CIRCUITPY_MSGPACK),1)
+SRC_PATTERNS += msgpack/%
+endif
 
 # All possible sources are listed here, and are filtered by SRC_PATTERNS in SRC_COMMON_HAL
 SRC_COMMON_HAL_ALL = \
@@ -396,6 +399,7 @@ $(filter $(SRC_PATTERNS), \
 	fontio/Glyph.c \
 	math/__init__.c \
 	microcontroller/RunMode.c \
+	msgpack/__init__.c \
 )
 
 SRC_BINDINGS_ENUMS += \
diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h
index bffab4f30f..3c4e5c2909 100644
--- a/py/circuitpy_mpconfig.h
+++ b/py/circuitpy_mpconfig.h
@@ -754,6 +754,13 @@ extern const struct _mp_obj_module_t wifi_module;
 #define WIFI_MODULE
 #endif
 
+#if CIRCUITPY_MSGPACK
+extern const struct _mp_obj_module_t msgpack_module;
+#define MSGPACK_MODULE { MP_ROM_QSTR(MP_QSTR_msgpack), MP_ROM_PTR(&msgpack_module) },
+#else
+#define MSGPACK_MODULE
+#endif
+
 // Define certain native modules with weak links so they can be replaced with Python
 // implementations. This list may grow over time.
 #define MICROPY_PORT_BUILTIN_MODULE_WEAK_LINKS \
@@ -839,6 +846,7 @@ extern const struct _mp_obj_module_t wifi_module;
     USTACK_MODULE \
     WATCHDOG_MODULE \
     WIFI_MODULE \
+    MSGPACK_MODULE \
 
 // If weak links are enabled, just include strong links in the main list of modules,
 // and also include the underscore alternate names.
diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index 08e7737180..38ca795849 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -296,3 +296,7 @@ CFLAGS += -DCIRCUITPY_WIFI=$(CIRCUITPY_WIFI)
 # Enabled micropython.native decorator (experimental)
 CIRCUITPY_ENABLE_MPY_NATIVE ?= 0
 CFLAGS += -DCIRCUITPY_ENABLE_MPY_NATIVE=$(CIRCUITPY_ENABLE_MPY_NATIVE)
+
+CIRCUITPY_MSGPACK ?= 0
+CFLAGS += -DCIRCUITPY_MSGPACK=$(CIRCUITPY_MSGPACK)
+
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
new file mode 100644
index 0000000000..74ab644702
--- /dev/null
+++ b/shared-bindings/msgpack/__init__.c
@@ -0,0 +1,391 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019  Bernhard Boser
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <stdio.h>
+
+#include "py/binary.h"
+#include "py/objarray.h"
+#include "py/objlist.h"
+#include "py/objstringio.h"
+#include "py/parsenum.h"
+#include "py/runtime.h"
+#include "py/stream.h"
+
+#include "supervisor/shared/translate.h"
+
+////////////////////////////////////////////////////////////////
+// stream management
+
+typedef struct _msgpack_stream_t {
+    mp_obj_t stream_obj;
+    mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode);
+    mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode);
+    int errcode;
+} msgpack_stream_t;
+
+STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) {
+    const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, flags);
+    msgpack_stream_t s = {stream_obj, stream_p->read, stream_p->write, 0};
+    return s;
+}
+
+////////////////////////////////////////////////////////////////
+// readers
+
+STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
+    if (size == 0) return;
+    mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode);
+    if (s->errcode != 0) {
+        mp_raise_OSError(s->errcode);
+    }
+    if (ret == 0) {
+        mp_raise_msg(&mp_type_EOFError, NULL);
+    }
+}
+
+STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) {
+    uint32_t res = 0;
+    read(s, &res, n_bytes);
+    return res;
+}
+
+size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
+    size_t n_bytes = 4;
+    switch (len_index) {
+        case 0:  n_bytes = 1;  break;
+        case 1:  n_bytes = 2;  break;
+        case 2:  n_bytes = 4;  break;
+    }
+    size_t res = 0;
+    read(s, &res, n_bytes);
+    return res;
+}
+
+////////////////////////////////////////////////////////////////
+// writers
+
+STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
+    mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode);
+    if (s->errcode != 0) {
+        mp_raise_OSError(s->errcode);
+    }
+    if (ret == 0) {
+        mp_raise_msg(&mp_type_EOFError, NULL);
+    }
+}
+
+STATIC void write_bytes(msgpack_stream_t *s, mp_uint_t n_bytes, uint32_t obj) {
+    write(s, &obj, n_bytes);
+}
+
+void write_size(msgpack_stream_t *s, uint8_t code, size_t size) {
+    if ((uint8_t)size == size) {
+        write_bytes(s, 1, code);
+        write_bytes(s, 1, size);
+    } else if ((uint16_t)size == size) {
+        write_bytes(s, 1, code+1);
+        write_bytes(s, 2, size);
+    } else {
+        write_bytes(s, 1, code+2);
+        write_bytes(s, 4, size);
+    }
+}
+
+////////////////////////////////////////////////////////////////
+// packers
+
+// This is a helper function to iterate through a dictionary.  The state of
+// the iteration is held in *cur and should be initialised with zero for the
+// first call.  Will return NULL when no more elements are available.
+STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) {
+    size_t max = dict->map.alloc;
+    mp_map_t *map = &dict->map;
+
+    for (size_t i = *cur; i < max; i++) {
+        if (MP_MAP_SLOT_IS_FILLED(map, i)) {
+            *cur = i + 1;
+            return &(map->table[i]);
+        }
+    }
+
+    return NULL;
+}
+
+
+STATIC void pack_int(msgpack_stream_t *s, int32_t x) {
+    if (x > -32 && x < 128) {
+        write_bytes(s, 1, x);
+    } else if ((int8_t)x == x) {
+        write_bytes(s, 1, 0xd0);
+        write_bytes(s, 1, x);
+    } else if ((int16_t)x == x) {
+        write_bytes(s, 1, 0xd1);
+        write_bytes(s, 2, x);
+    } else {
+        write_bytes(s, 1, 0xd2);
+        write_bytes(s, 4, x);
+    }
+}
+
+void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
+    write_size(s, 0xc4, len);
+    for (size_t i=0;  i<len;  i++) {
+        write_bytes(s, 1, data[i]);
+    }
+}
+
+void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
+    // size_t len = strlen(str);
+    if (len < 32) {
+        write_bytes(s, 1, 0b10100000 | (uint8_t)len);
+    } else {
+        write_size(s, 0xd9, len);
+    }
+    for (size_t l=0;  l<len;  l++) {
+        write_bytes(s, 1, str[l]);
+    }
+}
+
+void pack_array(msgpack_stream_t *s, size_t len) {
+    // only writes the header, manually write the objects after calling pack_array!
+    if (len < 16) {
+        write_bytes(s, 1, 0b10010000 | (uint8_t)len);
+    } else if (len < 0x10000) {
+        write_bytes(s, 1, 0xdc);
+        write_bytes(s, 2, len);
+    } else {
+        write_bytes(s, 1, 0xdd);
+        write_bytes(s, 4, len);
+    }
+}
+
+void pack_dict(msgpack_stream_t *s, size_t len) {
+    // only writes the header, manually write the objects after calling pack_array!
+    if (len < 16) {
+        write_bytes(s, 1, 0b10000000 | (uint8_t)len);
+    } else if (len < 0x10000) {
+        write_bytes(s, 1, 0xde);
+        write_bytes(s, 2, len);
+    } else {
+        write_bytes(s, 1, 0xdf);
+        write_bytes(s, 4, len);
+    }
+}
+
+// Write obj to stream in msgpack format
+STATIC void _pack(mp_obj_t obj, msgpack_stream_t *s) {
+    if (MP_OBJ_IS_SMALL_INT(obj)) {
+        // int
+        int32_t x = MP_OBJ_SMALL_INT_VALUE(obj);
+        pack_int(s, x);
+    } else if (MP_OBJ_IS_STR(obj)) {
+        // string
+        size_t len;
+        const char *data = mp_obj_str_get_data(obj, &len);
+        pack_str(s, data, len);
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_bytes)) {
+        // bytes
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer_raise(obj, &bufinfo, MP_BUFFER_READ);
+        pack_bin(s, bufinfo.buf, bufinfo.len);
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_tuple)) {
+        // tuple
+        mp_obj_tuple_t *self = MP_OBJ_TO_PTR(obj);
+        pack_array(s, self->len);
+        for (size_t i=0;  i<self->len;  i++) {
+            _pack(self->items[i], s);
+        }
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_list)) {
+        // list (layout differs from tuple)
+        mp_obj_list_t *self = MP_OBJ_TO_PTR(obj);
+        pack_array(s, self->len);
+        for (size_t i=0;  i<self->len;  i++) {
+            _pack(self->items[i], s);
+        }
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_dict)) {
+        // dict
+        mp_obj_dict_t *self = MP_OBJ_TO_PTR(obj);
+        pack_dict(s, self->map.used);
+        size_t cur = 0;
+        mp_map_elem_t *next = NULL;
+        while ((next = dict_iter_next(self, &cur)) != NULL) {
+            _pack(next->key, s);
+            _pack(next->value, s);
+        }
+    } else if (mp_obj_is_float(obj)) {
+        mp_float_t f = mp_obj_float_get(obj);
+        write_bytes(s, 1, 0xca);
+        write(s, &f, 4);
+    } else if (obj == mp_const_none) {
+        write_bytes(s, 1, 0xc0);
+    } else if (obj == mp_const_false) {
+        write_bytes(s, 1, 0xc2);
+    } else if (obj == mp_const_true) {
+        write_bytes(s, 1, 0xc3);
+    } else {
+        mp_raise_ValueError(translate("no packer"));
+    }
+}
+
+// Write obj to stream in msgpack format
+STATIC mp_obj_t mod_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) {
+    msgpack_stream_t s = get_stream(stream_obj, MP_STREAM_OP_WRITE);
+    _pack(obj, &s);
+    return mp_const_none;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_pack_obj, mod_msgpack_pack);
+
+////////////////////////////////////////////////////////////////
+// un-packers
+
+STATIC mp_obj_t _unpack(msgpack_stream_t *s) {
+    uint8_t code = read_bytes(s, 1);
+    if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) {
+        // int
+        return MP_OBJ_NEW_SMALL_INT((int8_t)code);
+    }
+    if ((code & 0b11100000) == 0b10100000) {
+        // str
+        size_t len = code & 0b11111;
+        // allocate on stack; len < 32
+        char str[len];
+        read(s, &str, len);
+        return mp_obj_new_str(str, len);
+    }
+    if ((code & 0b11110000) == 0b10010000) {
+        // array (tuple)
+        size_t len = code & 0b1111;
+        mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
+        for (size_t i=0; i<len; i++) {
+            t->items[i] = _unpack(s);
+        }
+        return MP_OBJ_FROM_PTR(t);
+    }
+    if ((code & 0b11110000) == 0b10000000) {
+        // map (dict)
+        size_t len = code & 0b1111;
+        mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
+        for (size_t i=0; i<len; i++) {
+            mp_obj_dict_store(d, _unpack(s), _unpack(s));
+        }
+        return MP_OBJ_FROM_PTR(d);
+    }
+    switch (code) {
+        case 0xc0: return mp_const_none;
+        case 0xc2: return mp_const_false;
+        case 0xc3: return mp_const_true;
+        case 0xc4:
+        case 0xc5:
+        case 0xc6: {
+            // bin 8, 16, 32
+            size_t size = read_size(s, code-0xc4);
+            vstr_t vstr;
+            vstr_init_len(&vstr, size);
+            byte *p = (byte*)vstr.buf;
+            read(s, p, size);
+            return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+        }
+        case 0xca: {
+            float f;
+            read(s, &f, 4);
+            return mp_obj_new_float(f);
+        }
+        case 0xd0:
+            return MP_OBJ_NEW_SMALL_INT((int8_t)read_bytes(s, 1));
+        case 0xd1:
+            return MP_OBJ_NEW_SMALL_INT((int16_t)read_bytes(s, 2));
+        case 0xd2:
+            return MP_OBJ_NEW_SMALL_INT((int32_t)read_bytes(s, 4));
+        case 0xd9:
+        case 0xda:
+        case 0xdb: {
+            // str 8, 16, 32
+            size_t size = read_size(s, code-0xd9);
+            vstr_t vstr;
+            vstr_init_len(&vstr, size);
+            byte *p = (byte*)vstr.buf;
+            read(s, p, size);
+            return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
+        }
+        case 0xde:
+        case 0xdf: {
+            // map 16 & 32
+            size_t len = read_size(s, code - 0xde + 1);
+            mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
+            for (size_t i=0; i<len; i++) {
+                mp_obj_dict_store(d, _unpack(s), _unpack(s));
+            }
+            return MP_OBJ_FROM_PTR(d);
+        }
+        case 0xdc:
+        case 0xdd: {
+            // array 16 & 32
+            size_t len = read_size(s, code - 0xdc + 1);
+            mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
+            for (size_t i=0; i<len; i++) {
+                t->items[i] = _unpack(s);
+            }
+            return MP_OBJ_FROM_PTR(t);
+        }
+        case 0xc1:      // never used
+        case 0xc7:      // ext 8
+        case 0xc8:      // ext 16
+        case 0xc9:      // ext 32
+        case 0xcb:      // float 64
+        case 0xcc:      // uint 8
+        case 0xcd:      // uint 16
+        case 0xce:      // uint 32
+        case 0xcf:      // uint 64
+        case 0xd3:      // int 64
+        case 0xd4:      // fixenxt 1
+        case 0xd5:      // fixenxt 2
+        case 0xd6:      // fixenxt 4
+        case 0xd7:      // fixenxt 8
+        case 0xd8:      // fixenxt 16
+        default:
+            mp_raise_ValueError(translate("no unpacker found"));
+    }
+}
+
+// Read msgpack encoded object from stream
+STATIC mp_obj_t mod_msgpack_unpack(mp_obj_t stream_obj) {
+    msgpack_stream_t s = get_stream(stream_obj, MP_STREAM_OP_READ);
+    return _unpack(&s);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_msgpack_unpack_obj, mod_msgpack_unpack);
+
+STATIC const mp_rom_map_elem_t msgpack_module_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_msgpack) },
+    { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&mod_msgpack_pack_obj) },
+    { MP_ROM_QSTR(MP_QSTR_unpack), MP_ROM_PTR(&mod_msgpack_unpack_obj) },
+};
+
+STATIC MP_DEFINE_CONST_DICT(msgpack_module_globals, msgpack_module_globals_table);
+
+const mp_obj_module_t msgpack_module = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t*)&msgpack_module_globals,
+};
diff --git a/shared-bindings/msgpack/__init__.h b/shared-bindings/msgpack/__init__.h
new file mode 100644
index 0000000000..4836cc62e7
--- /dev/null
+++ b/shared-bindings/msgpack/__init__.h
@@ -0,0 +1,34 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK___INIT___H
+#define MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK___INIT___H
+
+#include "py/obj.h"
+
+// Nothing now.
+
+#endif  // MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK___INIT___H

From 748472de7a0c2b18d88d868b1037252d57ab1bfb Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Sun, 8 Nov 2020 18:52:33 -0800
Subject: [PATCH 02/32] removed empty line at end of py/circuitpy_mpconfig.mk

---
 py/circuitpy_mpconfig.mk | 1 -
 1 file changed, 1 deletion(-)

diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index 38ca795849..4b6f4eb966 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -299,4 +299,3 @@ CFLAGS += -DCIRCUITPY_ENABLE_MPY_NATIVE=$(CIRCUITPY_ENABLE_MPY_NATIVE)
 
 CIRCUITPY_MSGPACK ?= 0
 CFLAGS += -DCIRCUITPY_MSGPACK=$(CIRCUITPY_MSGPACK)
-

From 513253bc3f22e5afa67155f7b4ca29a770929a3e Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Mon, 9 Nov 2020 18:09:56 -0800
Subject: [PATCH 03/32] moved logic to shared-module and added documentation

---
 ports/nrf/mpconfigport.mk          |   2 -
 py/circuitpy_defns.mk              |   1 +
 py/circuitpy_mpconfig.h            |   2 +-
 py/circuitpy_mpconfig.mk           |   2 +-
 shared-bindings/msgpack/__init__.c | 367 +++-------------------------
 shared-module/msgpack/__init__.c   | 372 +++++++++++++++++++++++++++++
 shared-module/msgpack/__init__.h   |  34 +++
 7 files changed, 437 insertions(+), 343 deletions(-)
 create mode 100644 shared-module/msgpack/__init__.c
 create mode 100644 shared-module/msgpack/__init__.h

diff --git a/ports/nrf/mpconfigport.mk b/ports/nrf/mpconfigport.mk
index 2ee182b7d5..9560064fbc 100644
--- a/ports/nrf/mpconfigport.mk
+++ b/ports/nrf/mpconfigport.mk
@@ -52,8 +52,6 @@ MCU_SUB_VARIANT = nrf52840
 # Fits on nrf52840 but space is tight on nrf52833.
 CIRCUITPY_AESIO ?= 1
 
-CIRCUITPY_MSGPACK ?= 1
-
 SD ?= s140
 SOFTDEV_VERSION ?= 6.1.0
 
diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk
index 144168c79e..8c452c2b03 100644
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@@ -465,6 +465,7 @@ SRC_SHARED_MODULE_ALL = \
 	memorymonitor/AllocationAlarm.c \
 	memorymonitor/AllocationSize.c \
 	network/__init__.c \
+	msgpack/__init__.c \
 	os/__init__.c \
 	random/__init__.c \
 	rgbmatrix/RGBMatrix.c \
diff --git a/py/circuitpy_mpconfig.h b/py/circuitpy_mpconfig.h
index 3c4e5c2909..64fedd506c 100644
--- a/py/circuitpy_mpconfig.h
+++ b/py/circuitpy_mpconfig.h
@@ -815,6 +815,7 @@ extern const struct _mp_obj_module_t msgpack_module;
     _EVE_MODULE \
     MEMORYMONITOR_MODULE \
     MICROCONTROLLER_MODULE \
+    MSGPACK_MODULE \
     NEOPIXEL_WRITE_MODULE \
     NETWORK_MODULE \
       SOCKET_MODULE \
@@ -846,7 +847,6 @@ extern const struct _mp_obj_module_t msgpack_module;
     USTACK_MODULE \
     WATCHDOG_MODULE \
     WIFI_MODULE \
-    MSGPACK_MODULE \
 
 // If weak links are enabled, just include strong links in the main list of modules,
 // and also include the underscore alternate names.
diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk
index 4b6f4eb966..348a22cd4b 100644
--- a/py/circuitpy_mpconfig.mk
+++ b/py/circuitpy_mpconfig.mk
@@ -297,5 +297,5 @@ CFLAGS += -DCIRCUITPY_WIFI=$(CIRCUITPY_WIFI)
 CIRCUITPY_ENABLE_MPY_NATIVE ?= 0
 CFLAGS += -DCIRCUITPY_ENABLE_MPY_NATIVE=$(CIRCUITPY_ENABLE_MPY_NATIVE)
 
-CIRCUITPY_MSGPACK ?= 0
+CIRCUITPY_MSGPACK ?= $(CIRCUITPY_FULL_BUILD)
 CFLAGS += -DCIRCUITPY_MSGPACK=$(CIRCUITPY_MSGPACK)
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 74ab644702..6329c50269 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -24,359 +24,48 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
+#include "shared-bindings/msgpack/__init__.h"
+#include "shared-module/msgpack/__init__.h"
 
-#include "py/binary.h"
-#include "py/objarray.h"
-#include "py/objlist.h"
-#include "py/objstringio.h"
-#include "py/parsenum.h"
-#include "py/runtime.h"
-#include "py/stream.h"
+//| """Pack object in msgpack format
+//|
+//| The msgpack format is similar to json, except that the encoded data is binary.
+//| See https://msgpack.org for details.
+//|
+//| Example:
+//|    import msgpack
+//|    from io import StringIO
+//|    
+//|    s = StringIO()
+//|    msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, s)
+//|    s.seek(0)
+//|    print(msgpack.unpack(s))"""
+//|    
 
-#include "supervisor/shared/translate.h"
+//| def pack(obj, stream):
+//|     """Pack obj to stream."""
+//|     ...
+//|
 
-////////////////////////////////////////////////////////////////
-// stream management
-
-typedef struct _msgpack_stream_t {
-    mp_obj_t stream_obj;
-    mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode);
-    mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode);
-    int errcode;
-} msgpack_stream_t;
-
-STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) {
-    const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, flags);
-    msgpack_stream_t s = {stream_obj, stream_p->read, stream_p->write, 0};
-    return s;
-}
-
-////////////////////////////////////////////////////////////////
-// readers
-
-STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
-    if (size == 0) return;
-    mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode);
-    if (s->errcode != 0) {
-        mp_raise_OSError(s->errcode);
-    }
-    if (ret == 0) {
-        mp_raise_msg(&mp_type_EOFError, NULL);
-    }
-}
-
-STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) {
-    uint32_t res = 0;
-    read(s, &res, n_bytes);
-    return res;
-}
-
-size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
-    size_t n_bytes = 4;
-    switch (len_index) {
-        case 0:  n_bytes = 1;  break;
-        case 1:  n_bytes = 2;  break;
-        case 2:  n_bytes = 4;  break;
-    }
-    size_t res = 0;
-    read(s, &res, n_bytes);
-    return res;
-}
-
-////////////////////////////////////////////////////////////////
-// writers
-
-STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
-    mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode);
-    if (s->errcode != 0) {
-        mp_raise_OSError(s->errcode);
-    }
-    if (ret == 0) {
-        mp_raise_msg(&mp_type_EOFError, NULL);
-    }
-}
-
-STATIC void write_bytes(msgpack_stream_t *s, mp_uint_t n_bytes, uint32_t obj) {
-    write(s, &obj, n_bytes);
-}
-
-void write_size(msgpack_stream_t *s, uint8_t code, size_t size) {
-    if ((uint8_t)size == size) {
-        write_bytes(s, 1, code);
-        write_bytes(s, 1, size);
-    } else if ((uint16_t)size == size) {
-        write_bytes(s, 1, code+1);
-        write_bytes(s, 2, size);
-    } else {
-        write_bytes(s, 1, code+2);
-        write_bytes(s, 4, size);
-    }
-}
-
-////////////////////////////////////////////////////////////////
-// packers
-
-// This is a helper function to iterate through a dictionary.  The state of
-// the iteration is held in *cur and should be initialised with zero for the
-// first call.  Will return NULL when no more elements are available.
-STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) {
-    size_t max = dict->map.alloc;
-    mp_map_t *map = &dict->map;
-
-    for (size_t i = *cur; i < max; i++) {
-        if (MP_MAP_SLOT_IS_FILLED(map, i)) {
-            *cur = i + 1;
-            return &(map->table[i]);
-        }
-    }
-
-    return NULL;
-}
-
-
-STATIC void pack_int(msgpack_stream_t *s, int32_t x) {
-    if (x > -32 && x < 128) {
-        write_bytes(s, 1, x);
-    } else if ((int8_t)x == x) {
-        write_bytes(s, 1, 0xd0);
-        write_bytes(s, 1, x);
-    } else if ((int16_t)x == x) {
-        write_bytes(s, 1, 0xd1);
-        write_bytes(s, 2, x);
-    } else {
-        write_bytes(s, 1, 0xd2);
-        write_bytes(s, 4, x);
-    }
-}
-
-void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
-    write_size(s, 0xc4, len);
-    for (size_t i=0;  i<len;  i++) {
-        write_bytes(s, 1, data[i]);
-    }
-}
-
-void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
-    // size_t len = strlen(str);
-    if (len < 32) {
-        write_bytes(s, 1, 0b10100000 | (uint8_t)len);
-    } else {
-        write_size(s, 0xd9, len);
-    }
-    for (size_t l=0;  l<len;  l++) {
-        write_bytes(s, 1, str[l]);
-    }
-}
-
-void pack_array(msgpack_stream_t *s, size_t len) {
-    // only writes the header, manually write the objects after calling pack_array!
-    if (len < 16) {
-        write_bytes(s, 1, 0b10010000 | (uint8_t)len);
-    } else if (len < 0x10000) {
-        write_bytes(s, 1, 0xdc);
-        write_bytes(s, 2, len);
-    } else {
-        write_bytes(s, 1, 0xdd);
-        write_bytes(s, 4, len);
-    }
-}
-
-void pack_dict(msgpack_stream_t *s, size_t len) {
-    // only writes the header, manually write the objects after calling pack_array!
-    if (len < 16) {
-        write_bytes(s, 1, 0b10000000 | (uint8_t)len);
-    } else if (len < 0x10000) {
-        write_bytes(s, 1, 0xde);
-        write_bytes(s, 2, len);
-    } else {
-        write_bytes(s, 1, 0xdf);
-        write_bytes(s, 4, len);
-    }
-}
-
-// Write obj to stream in msgpack format
-STATIC void _pack(mp_obj_t obj, msgpack_stream_t *s) {
-    if (MP_OBJ_IS_SMALL_INT(obj)) {
-        // int
-        int32_t x = MP_OBJ_SMALL_INT_VALUE(obj);
-        pack_int(s, x);
-    } else if (MP_OBJ_IS_STR(obj)) {
-        // string
-        size_t len;
-        const char *data = mp_obj_str_get_data(obj, &len);
-        pack_str(s, data, len);
-    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_bytes)) {
-        // bytes
-        mp_buffer_info_t bufinfo;
-        mp_get_buffer_raise(obj, &bufinfo, MP_BUFFER_READ);
-        pack_bin(s, bufinfo.buf, bufinfo.len);
-    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_tuple)) {
-        // tuple
-        mp_obj_tuple_t *self = MP_OBJ_TO_PTR(obj);
-        pack_array(s, self->len);
-        for (size_t i=0;  i<self->len;  i++) {
-            _pack(self->items[i], s);
-        }
-    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_list)) {
-        // list (layout differs from tuple)
-        mp_obj_list_t *self = MP_OBJ_TO_PTR(obj);
-        pack_array(s, self->len);
-        for (size_t i=0;  i<self->len;  i++) {
-            _pack(self->items[i], s);
-        }
-    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_dict)) {
-        // dict
-        mp_obj_dict_t *self = MP_OBJ_TO_PTR(obj);
-        pack_dict(s, self->map.used);
-        size_t cur = 0;
-        mp_map_elem_t *next = NULL;
-        while ((next = dict_iter_next(self, &cur)) != NULL) {
-            _pack(next->key, s);
-            _pack(next->value, s);
-        }
-    } else if (mp_obj_is_float(obj)) {
-        mp_float_t f = mp_obj_float_get(obj);
-        write_bytes(s, 1, 0xca);
-        write(s, &f, 4);
-    } else if (obj == mp_const_none) {
-        write_bytes(s, 1, 0xc0);
-    } else if (obj == mp_const_false) {
-        write_bytes(s, 1, 0xc2);
-    } else if (obj == mp_const_true) {
-        write_bytes(s, 1, 0xc3);
-    } else {
-        mp_raise_ValueError(translate("no packer"));
-    }
-}
-
-// Write obj to stream in msgpack format
 STATIC mp_obj_t mod_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) {
-    msgpack_stream_t s = get_stream(stream_obj, MP_STREAM_OP_WRITE);
-    _pack(obj, &s);
+    common_hal_msgpack_pack(obj, stream_obj);
     return mp_const_none;
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_pack_obj, mod_msgpack_pack);
 
-////////////////////////////////////////////////////////////////
-// un-packers
 
-STATIC mp_obj_t _unpack(msgpack_stream_t *s) {
-    uint8_t code = read_bytes(s, 1);
-    if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) {
-        // int
-        return MP_OBJ_NEW_SMALL_INT((int8_t)code);
-    }
-    if ((code & 0b11100000) == 0b10100000) {
-        // str
-        size_t len = code & 0b11111;
-        // allocate on stack; len < 32
-        char str[len];
-        read(s, &str, len);
-        return mp_obj_new_str(str, len);
-    }
-    if ((code & 0b11110000) == 0b10010000) {
-        // array (tuple)
-        size_t len = code & 0b1111;
-        mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
-        for (size_t i=0; i<len; i++) {
-            t->items[i] = _unpack(s);
-        }
-        return MP_OBJ_FROM_PTR(t);
-    }
-    if ((code & 0b11110000) == 0b10000000) {
-        // map (dict)
-        size_t len = code & 0b1111;
-        mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
-        for (size_t i=0; i<len; i++) {
-            mp_obj_dict_store(d, _unpack(s), _unpack(s));
-        }
-        return MP_OBJ_FROM_PTR(d);
-    }
-    switch (code) {
-        case 0xc0: return mp_const_none;
-        case 0xc2: return mp_const_false;
-        case 0xc3: return mp_const_true;
-        case 0xc4:
-        case 0xc5:
-        case 0xc6: {
-            // bin 8, 16, 32
-            size_t size = read_size(s, code-0xc4);
-            vstr_t vstr;
-            vstr_init_len(&vstr, size);
-            byte *p = (byte*)vstr.buf;
-            read(s, p, size);
-            return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
-        }
-        case 0xca: {
-            float f;
-            read(s, &f, 4);
-            return mp_obj_new_float(f);
-        }
-        case 0xd0:
-            return MP_OBJ_NEW_SMALL_INT((int8_t)read_bytes(s, 1));
-        case 0xd1:
-            return MP_OBJ_NEW_SMALL_INT((int16_t)read_bytes(s, 2));
-        case 0xd2:
-            return MP_OBJ_NEW_SMALL_INT((int32_t)read_bytes(s, 4));
-        case 0xd9:
-        case 0xda:
-        case 0xdb: {
-            // str 8, 16, 32
-            size_t size = read_size(s, code-0xd9);
-            vstr_t vstr;
-            vstr_init_len(&vstr, size);
-            byte *p = (byte*)vstr.buf;
-            read(s, p, size);
-            return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
-        }
-        case 0xde:
-        case 0xdf: {
-            // map 16 & 32
-            size_t len = read_size(s, code - 0xde + 1);
-            mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
-            for (size_t i=0; i<len; i++) {
-                mp_obj_dict_store(d, _unpack(s), _unpack(s));
-            }
-            return MP_OBJ_FROM_PTR(d);
-        }
-        case 0xdc:
-        case 0xdd: {
-            // array 16 & 32
-            size_t len = read_size(s, code - 0xdc + 1);
-            mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
-            for (size_t i=0; i<len; i++) {
-                t->items[i] = _unpack(s);
-            }
-            return MP_OBJ_FROM_PTR(t);
-        }
-        case 0xc1:      // never used
-        case 0xc7:      // ext 8
-        case 0xc8:      // ext 16
-        case 0xc9:      // ext 32
-        case 0xcb:      // float 64
-        case 0xcc:      // uint 8
-        case 0xcd:      // uint 16
-        case 0xce:      // uint 32
-        case 0xcf:      // uint 64
-        case 0xd3:      // int 64
-        case 0xd4:      // fixenxt 1
-        case 0xd5:      // fixenxt 2
-        case 0xd6:      // fixenxt 4
-        case 0xd7:      // fixenxt 8
-        case 0xd8:      // fixenxt 16
-        default:
-            mp_raise_ValueError(translate("no unpacker found"));
-    }
-}
+//| def pack(obj, stream) -> obj:
+//|     """Unpack and return one object (in msgpack format) from stream.
+//|     Call repeatedly to read multiple objects from the stream."""
+//|     ...
+//|
 
-// Read msgpack encoded object from stream
 STATIC mp_obj_t mod_msgpack_unpack(mp_obj_t stream_obj) {
-    msgpack_stream_t s = get_stream(stream_obj, MP_STREAM_OP_READ);
-    return _unpack(&s);
+    return common_hal_msgpack_unpack(stream_obj);
 }
 STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_msgpack_unpack_obj, mod_msgpack_unpack);
 
+
 STATIC const mp_rom_map_elem_t msgpack_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_msgpack) },
     { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&mod_msgpack_pack_obj) },
diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
new file mode 100644
index 0000000000..4a969901d1
--- /dev/null
+++ b/shared-module/msgpack/__init__.c
@@ -0,0 +1,372 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019  Bernhard Boser
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <stdio.h>
+
+#include "py/binary.h"
+#include "py/objarray.h"
+#include "py/objlist.h"
+#include "py/objstringio.h"
+#include "py/parsenum.h"
+#include "py/runtime.h"
+#include "py/stream.h"
+
+#include "supervisor/shared/translate.h"
+
+////////////////////////////////////////////////////////////////
+// stream management
+
+typedef struct _msgpack_stream_t {
+    mp_obj_t stream_obj;
+    mp_uint_t (*read)(mp_obj_t obj, void *buf, mp_uint_t size, int *errcode);
+    mp_uint_t (*write)(mp_obj_t obj, const void *buf, mp_uint_t size, int *errcode);
+    int errcode;
+} msgpack_stream_t;
+
+STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) {
+    const mp_stream_p_t *stream_p = mp_get_stream_raise(stream_obj, flags);
+    msgpack_stream_t s = {stream_obj, stream_p->read, stream_p->write, 0};
+    return s;
+}
+
+////////////////////////////////////////////////////////////////
+// readers
+
+STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
+    if (size == 0) return;
+    mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode);
+    if (s->errcode != 0) {
+        mp_raise_OSError(s->errcode);
+    }
+    if (ret == 0) {
+        mp_raise_msg(&mp_type_EOFError, NULL);
+    }
+}
+
+STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) {
+    uint32_t res = 0;
+    read(s, &res, n_bytes);
+    return res;
+}
+
+size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
+    size_t n_bytes = 4;
+    switch (len_index) {
+        case 0:  n_bytes = 1;  break;
+        case 1:  n_bytes = 2;  break;
+        case 2:  n_bytes = 4;  break;
+    }
+    size_t res = 0;
+    read(s, &res, n_bytes);
+    return res;
+}
+
+////////////////////////////////////////////////////////////////
+// writers
+
+STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
+    mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode);
+    if (s->errcode != 0) {
+        mp_raise_OSError(s->errcode);
+    }
+    if (ret == 0) {
+        mp_raise_msg(&mp_type_EOFError, NULL);
+    }
+}
+
+STATIC void write_bytes(msgpack_stream_t *s, mp_uint_t n_bytes, uint32_t obj) {
+    write(s, &obj, n_bytes);
+}
+
+void write_size(msgpack_stream_t *s, uint8_t code, size_t size) {
+    if ((uint8_t)size == size) {
+        write_bytes(s, 1, code);
+        write_bytes(s, 1, size);
+    } else if ((uint16_t)size == size) {
+        write_bytes(s, 1, code+1);
+        write_bytes(s, 2, size);
+    } else {
+        write_bytes(s, 1, code+2);
+        write_bytes(s, 4, size);
+    }
+}
+
+////////////////////////////////////////////////////////////////
+// packers
+
+// This is a helper function to iterate through a dictionary.  The state of
+// the iteration is held in *cur and should be initialised with zero for the
+// first call.  Will return NULL when no more elements are available.
+STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) {
+    size_t max = dict->map.alloc;
+    mp_map_t *map = &dict->map;
+
+    for (size_t i = *cur; i < max; i++) {
+        if (MP_MAP_SLOT_IS_FILLED(map, i)) {
+            *cur = i + 1;
+            return &(map->table[i]);
+        }
+    }
+
+    return NULL;
+}
+
+STATIC void pack_int(msgpack_stream_t *s, int32_t x) {
+    if (x > -32 && x < 128) {
+        write_bytes(s, 1, x);
+    } else if ((int8_t)x == x) {
+        write_bytes(s, 1, 0xd0);
+        write_bytes(s, 1, x);
+    } else if ((int16_t)x == x) {
+        write_bytes(s, 1, 0xd1);
+        write_bytes(s, 2, x);
+    } else {
+        write_bytes(s, 1, 0xd2);
+        write_bytes(s, 4, x);
+    }
+}
+
+void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
+    write_size(s, 0xc4, len);
+    for (size_t i=0;  i<len;  i++) {
+        write_bytes(s, 1, data[i]);
+    }
+}
+
+void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
+    // size_t len = strlen(str);
+    if (len < 32) {
+        write_bytes(s, 1, 0b10100000 | (uint8_t)len);
+    } else {
+        write_size(s, 0xd9, len);
+    }
+    for (size_t l=0;  l<len;  l++) {
+        write_bytes(s, 1, str[l]);
+    }
+}
+
+void pack_array(msgpack_stream_t *s, size_t len) {
+    // only writes the header, manually write the objects after calling pack_array!
+    if (len < 16) {
+        write_bytes(s, 1, 0b10010000 | (uint8_t)len);
+    } else if (len < 0x10000) {
+        write_bytes(s, 1, 0xdc);
+        write_bytes(s, 2, len);
+    } else {
+        write_bytes(s, 1, 0xdd);
+        write_bytes(s, 4, len);
+    }
+}
+
+void pack_dict(msgpack_stream_t *s, size_t len) {
+    // only writes the header, manually write the objects after calling pack_array!
+    if (len < 16) {
+        write_bytes(s, 1, 0b10000000 | (uint8_t)len);
+    } else if (len < 0x10000) {
+        write_bytes(s, 1, 0xde);
+        write_bytes(s, 2, len);
+    } else {
+        write_bytes(s, 1, 0xdf);
+        write_bytes(s, 4, len);
+    }
+}
+
+void pack(mp_obj_t obj, msgpack_stream_t *s) {
+    if (MP_OBJ_IS_SMALL_INT(obj)) {
+        // int
+        int32_t x = MP_OBJ_SMALL_INT_VALUE(obj);
+        pack_int(s, x);
+    } else if (MP_OBJ_IS_STR(obj)) {
+        // string
+        size_t len;
+        const char *data = mp_obj_str_get_data(obj, &len);
+        pack_str(s, data, len);
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_bytes)) {
+        // bytes
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer_raise(obj, &bufinfo, MP_BUFFER_READ);
+        pack_bin(s, bufinfo.buf, bufinfo.len);
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_tuple)) {
+        // tuple
+        mp_obj_tuple_t *self = MP_OBJ_TO_PTR(obj);
+        pack_array(s, self->len);
+        for (size_t i=0;  i<self->len;  i++) {
+            pack(self->items[i], s);
+        }
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_list)) {
+        // list (layout differs from tuple)
+        mp_obj_list_t *self = MP_OBJ_TO_PTR(obj);
+        pack_array(s, self->len);
+        for (size_t i=0;  i<self->len;  i++) {
+            pack(self->items[i], s);
+        }
+    } else if (MP_OBJ_IS_TYPE(obj, &mp_type_dict)) {
+        // dict
+        mp_obj_dict_t *self = MP_OBJ_TO_PTR(obj);
+        pack_dict(s, self->map.used);
+        size_t cur = 0;
+        mp_map_elem_t *next = NULL;
+        while ((next = dict_iter_next(self, &cur)) != NULL) {
+            pack(next->key, s);
+            pack(next->value, s);
+        }
+    } else if (mp_obj_is_float(obj)) {
+        mp_float_t f = mp_obj_float_get(obj);
+        write_bytes(s, 1, 0xca);
+        write(s, &f, 4);
+    } else if (obj == mp_const_none) {
+        write_bytes(s, 1, 0xc0);
+    } else if (obj == mp_const_false) {
+        write_bytes(s, 1, 0xc2);
+    } else if (obj == mp_const_true) {
+        write_bytes(s, 1, 0xc3);
+    } else {
+        mp_raise_ValueError(translate("no packer"));
+    }
+}
+
+////////////////////////////////////////////////////////////////
+// unpacker
+
+mp_obj_t unpack(msgpack_stream_t *s) {
+    uint8_t code = read_bytes(s, 1);
+    if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) {
+        // int
+        return MP_OBJ_NEW_SMALL_INT((int8_t)code);
+    }
+    if ((code & 0b11100000) == 0b10100000) {
+        // str
+        size_t len = code & 0b11111;
+        // allocate on stack; len < 32
+        char str[len];
+        read(s, &str, len);
+        return mp_obj_new_str(str, len);
+    }
+    if ((code & 0b11110000) == 0b10010000) {
+        // array (tuple)
+        size_t len = code & 0b1111;
+        mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
+        for (size_t i=0; i<len; i++) {
+            t->items[i] = unpack(s);
+        }
+        return MP_OBJ_FROM_PTR(t);
+    }
+    if ((code & 0b11110000) == 0b10000000) {
+        // map (dict)
+        size_t len = code & 0b1111;
+        mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
+        for (size_t i=0; i<len; i++) {
+            mp_obj_dict_store(d, unpack(s), unpack(s));
+        }
+        return MP_OBJ_FROM_PTR(d);
+    }
+    switch (code) {
+        case 0xc0: return mp_const_none;
+        case 0xc2: return mp_const_false;
+        case 0xc3: return mp_const_true;
+        case 0xc4:
+        case 0xc5:
+        case 0xc6: {
+            // bin 8, 16, 32
+            size_t size = read_size(s, code-0xc4);
+            vstr_t vstr;
+            vstr_init_len(&vstr, size);
+            byte *p = (byte*)vstr.buf;
+            read(s, p, size);
+            return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+        }
+        case 0xca: {
+            float f;
+            read(s, &f, 4);
+            return mp_obj_new_float(f);
+        }
+        case 0xd0:
+            return MP_OBJ_NEW_SMALL_INT((int8_t)read_bytes(s, 1));
+        case 0xd1:
+            return MP_OBJ_NEW_SMALL_INT((int16_t)read_bytes(s, 2));
+        case 0xd2:
+            return MP_OBJ_NEW_SMALL_INT((int32_t)read_bytes(s, 4));
+        case 0xd9:
+        case 0xda:
+        case 0xdb: {
+            // str 8, 16, 32
+            size_t size = read_size(s, code-0xd9);
+            vstr_t vstr;
+            vstr_init_len(&vstr, size);
+            byte *p = (byte*)vstr.buf;
+            read(s, p, size);
+            return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
+        }
+        case 0xde:
+        case 0xdf: {
+            // map 16 & 32
+            size_t len = read_size(s, code - 0xde + 1);
+            mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
+            for (size_t i=0; i<len; i++) {
+                mp_obj_dict_store(d, unpack(s), unpack(s));
+            }
+            return MP_OBJ_FROM_PTR(d);
+        }
+        case 0xdc:
+        case 0xdd: {
+            // array 16 & 32
+            size_t len = read_size(s, code - 0xdc + 1);
+            mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
+            for (size_t i=0; i<len; i++) {
+                t->items[i] = unpack(s);
+            }
+            return MP_OBJ_FROM_PTR(t);
+        }
+        case 0xc1:      // never used
+        case 0xc7:      // ext 8
+        case 0xc8:      // ext 16
+        case 0xc9:      // ext 32
+        case 0xcb:      // float 64
+        case 0xcc:      // uint 8
+        case 0xcd:      // uint 16
+        case 0xce:      // uint 32
+        case 0xcf:      // uint 64
+        case 0xd3:      // int 64
+        case 0xd4:      // fixenxt 1
+        case 0xd5:      // fixenxt 2
+        case 0xd6:      // fixenxt 4
+        case 0xd7:      // fixenxt 8
+        case 0xd8:      // fixenxt 16
+        default:
+            mp_raise_ValueError(translate("no unpacker found"));
+    }
+}
+
+void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) {
+    msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE);
+    pack(obj, &stream);
+}
+
+mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj) {
+    msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE);
+    return unpack(&stream);
+}
+
diff --git a/shared-module/msgpack/__init__.h b/shared-module/msgpack/__init__.h
new file mode 100644
index 0000000000..0a5e7852ad
--- /dev/null
+++ b/shared-module/msgpack/__init__.h
@@ -0,0 +1,34 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2017 Scott Shawcroft for Adafruit Industries
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_SHARED_MODULE_MSGPACK___INIT___H
+#define MICROPY_INCLUDED_SHARED_MODULE_MSGPACK___INIT___H
+
+#include "py/stream.h"
+
+void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj);
+mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj);
+
+#endif

From e19782633a88ac5b076119d506ddef1d35728e4d Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Mon, 9 Nov 2020 18:12:55 -0800
Subject: [PATCH 04/32] removed blanks to make pre-commit check happy

---
 shared-bindings/msgpack/__init__.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 6329c50269..b53fb93998 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -35,12 +35,12 @@
 //| Example:
 //|    import msgpack
 //|    from io import StringIO
-//|    
+//|
 //|    s = StringIO()
 //|    msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, s)
 //|    s.seek(0)
 //|    print(msgpack.unpack(s))"""
-//|    
+//|
 
 //| def pack(obj, stream):
 //|     """Pack obj to stream."""

From 912691e0d28ecbef563610dc328f6b08c61b31b0 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Mon, 9 Nov 2020 18:16:22 -0800
Subject: [PATCH 05/32] removed blanks line at end of file

---
 shared-module/msgpack/__init__.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index 4a969901d1..437469978b 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -369,4 +369,3 @@ mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj) {
     msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE);
     return unpack(&stream);
 }
-

From f38bd59c6bbf3cb2f5e4417c676a12444e39f11b Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Mon, 9 Nov 2020 18:29:31 -0800
Subject: [PATCH 06/32] added signatures to documenation

---
 shared-bindings/msgpack/__init__.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index b53fb93998..720d88109a 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -42,7 +42,7 @@
 //|    print(msgpack.unpack(s))"""
 //|
 
-//| def pack(obj, stream):
+//| def pack(obj: Any, buffer: WriteableBuffer) -> None:
 //|     """Pack obj to stream."""
 //|     ...
 //|
@@ -54,7 +54,7 @@ STATIC mp_obj_t mod_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) {
 STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_pack_obj, mod_msgpack_pack);
 
 
-//| def pack(obj, stream) -> obj:
+//| def unpack(buffer: ReadableBuffer) -> Any:
 //|     """Unpack and return one object (in msgpack format) from stream.
 //|     Call repeatedly to read multiple objects from the stream."""
 //|     ...

From f91932d78374bec0fe6225f53a487919a956b76f Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Wed, 11 Nov 2020 16:37:15 -0800
Subject: [PATCH 07/32] make translate

---
 locale/circuitpython.pot | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot
index 274d92168a..51feb16c35 100644
--- a/locale/circuitpython.pot
+++ b/locale/circuitpython.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-11-24 15:40-0500\n"
+"POT-Creation-Date: 2020-12-01 18:38-0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -3050,6 +3050,10 @@ msgstr ""
 msgid "no module named '%q'"
 msgstr ""
 
+#: shared-module/msgpack/__init__.c
+msgid "no packer"
+msgstr ""
+
 #: shared-bindings/displayio/FourWire.c shared-bindings/displayio/I2CDisplay.c
 #: shared-bindings/displayio/ParallelBus.c
 msgid "no reset pin available"
@@ -3063,6 +3067,10 @@ msgstr ""
 msgid "no such attribute"
 msgstr ""
 
+#: shared-module/msgpack/__init__.c
+msgid "no unpacker found"
+msgstr ""
+
 #: ports/nrf/common-hal/_bleio/Connection.c
 msgid "non-UUID found in service_uuids_whitelist"
 msgstr ""

From 44bc75af37a464eb6e5ba26bab19f2150d58044d Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Thu, 12 Nov 2020 12:28:40 -0800
Subject: [PATCH 08/32] output little endian; update example

---
 shared-bindings/msgpack/__init__.c |  10 +-
 shared-module/msgpack/__init__.c   | 147 ++++++++++++++++++++---------
 2 files changed, 106 insertions(+), 51 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 720d88109a..a222177f74 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -34,12 +34,12 @@
 //|
 //| Example:
 //|    import msgpack
-//|    from io import StringIO
+//|    from io import BytesIO
 //|
-//|    s = StringIO()
-//|    msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, s)
-//|    s.seek(0)
-//|    print(msgpack.unpack(s))"""
+//|    b = BytesIO()
+//|    msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, b)
+//|    b.seek(0)
+//|    print(msgpack.unpack(b))"""
 //|
 
 //| def pack(obj: Any, buffer: WriteableBuffer) -> None:
diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index 437469978b..505c1c9697 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -25,6 +25,7 @@
  */
 
 #include <stdio.h>
+#include <inttypes.h>
 
 #include "py/binary.h"
 #include "py/objarray.h"
@@ -66,13 +67,37 @@ STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
     }
 }
 
+/*
 STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) {
     uint32_t res = 0;
     read(s, &res, n_bytes);
     return res;
 }
+*/
+STATIC uint8_t read1(msgpack_stream_t *s) {
+    uint8_t res = 0;
+    read(s, &res, 1);
+    return res;
+}
 
-size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
+STATIC uint16_t read2(msgpack_stream_t *s) {
+    uint16_t res = 0;
+    read(s, &res, 2);
+    int n = 1;
+    if (*(char *)&n == 1) res = __builtin_bswap16(res);
+    return res;
+}
+
+STATIC uint32_t read4(msgpack_stream_t *s) {
+    uint32_t res = 0;
+    read(s, &res, 4);
+    int n = 1;
+    if (*(char *)&n == 1) res = __builtin_bswap32(res);
+    return res;
+}
+
+/*
+STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
     size_t n_bytes = 4;
     switch (len_index) {
         case 0:  n_bytes = 1;  break;
@@ -83,6 +108,17 @@ size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
     read(s, &res, n_bytes);
     return res;
 }
+*/
+
+STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
+    size_t res;
+    switch (len_index) {
+        case 0:  res = (size_t)read1(s);  break;
+        case 1:  res = (size_t)read2(s);  break;
+        case 2:  res = (size_t)read4(s);  break;
+    }
+    return res;
+}
 
 ////////////////////////////////////////////////////////////////
 // writers
@@ -97,20 +133,33 @@ STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
     }
 }
 
-STATIC void write_bytes(msgpack_stream_t *s, mp_uint_t n_bytes, uint32_t obj) {
-    write(s, &obj, n_bytes);
+STATIC void write1(msgpack_stream_t *s, uint8_t obj) {
+    write(s, &obj, 1);
 }
 
-void write_size(msgpack_stream_t *s, uint8_t code, size_t size) {
+STATIC void write2(msgpack_stream_t *s, uint16_t obj) {
+    int n = 1;
+    if (*(char *)&n == 1) obj = __builtin_bswap16(obj);
+    write(s, &obj, 2);
+}
+
+STATIC void write4(msgpack_stream_t *s, uint32_t obj) {
+    int n = 1;
+    if (*(char *)&n == 1) obj = __builtin_bswap32(obj);
+    write(s, &obj, 4);
+}
+
+// compute and write msgpack size code (array structures)
+STATIC void write_size(msgpack_stream_t *s, uint8_t code, size_t size) {
     if ((uint8_t)size == size) {
-        write_bytes(s, 1, code);
-        write_bytes(s, 1, size);
+        write1(s, code);
+        write1(s, size);
     } else if ((uint16_t)size == size) {
-        write_bytes(s, 1, code+1);
-        write_bytes(s, 2, size);
+        write1(s, code+1);
+        write2(s, size);
     } else {
-        write_bytes(s, 1, code+2);
-        write_bytes(s, 4, size);
+        write1(s, code+2);
+        write4(s, size);
     }
 }
 
@@ -136,61 +185,61 @@ STATIC mp_map_elem_t *dict_iter_next(mp_obj_dict_t *dict, size_t *cur) {
 
 STATIC void pack_int(msgpack_stream_t *s, int32_t x) {
     if (x > -32 && x < 128) {
-        write_bytes(s, 1, x);
+        write1(s, x);
     } else if ((int8_t)x == x) {
-        write_bytes(s, 1, 0xd0);
-        write_bytes(s, 1, x);
+        write1(s, 0xd0);
+        write1(s, x);
     } else if ((int16_t)x == x) {
-        write_bytes(s, 1, 0xd1);
-        write_bytes(s, 2, x);
+        write1(s, 0xd1);
+        write2(s, x);
     } else {
-        write_bytes(s, 1, 0xd2);
-        write_bytes(s, 4, x);
+        write1(s, 0xd2);
+        write4(s, x);
     }
 }
 
 void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
     write_size(s, 0xc4, len);
     for (size_t i=0;  i<len;  i++) {
-        write_bytes(s, 1, data[i]);
+        write1(s, data[i]);
     }
 }
 
 void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
     // size_t len = strlen(str);
     if (len < 32) {
-        write_bytes(s, 1, 0b10100000 | (uint8_t)len);
+        write1(s, 0b10100000 | (uint8_t)len);
     } else {
         write_size(s, 0xd9, len);
     }
     for (size_t l=0;  l<len;  l++) {
-        write_bytes(s, 1, str[l]);
+        write1(s, str[l]);
     }
 }
 
 void pack_array(msgpack_stream_t *s, size_t len) {
     // only writes the header, manually write the objects after calling pack_array!
     if (len < 16) {
-        write_bytes(s, 1, 0b10010000 | (uint8_t)len);
+        write1(s, 0b10010000 | (uint8_t)len);
     } else if (len < 0x10000) {
-        write_bytes(s, 1, 0xdc);
-        write_bytes(s, 2, len);
+        write1(s, 0xdc);
+        write2(s, len);
     } else {
-        write_bytes(s, 1, 0xdd);
-        write_bytes(s, 4, len);
+        write1(s, 0xdd);
+        write4(s, len);
     }
 }
 
 void pack_dict(msgpack_stream_t *s, size_t len) {
     // only writes the header, manually write the objects after calling pack_array!
     if (len < 16) {
-        write_bytes(s, 1, 0b10000000 | (uint8_t)len);
+        write1(s, 0b10000000 | (uint8_t)len);
     } else if (len < 0x10000) {
-        write_bytes(s, 1, 0xde);
-        write_bytes(s, 2, len);
+        write1(s, 0xde);
+        write2(s, len);
     } else {
-        write_bytes(s, 1, 0xdf);
-        write_bytes(s, 4, len);
+        write1(s, 0xdf);
+        write4(s, len);
     }
 }
 
@@ -234,15 +283,17 @@ void pack(mp_obj_t obj, msgpack_stream_t *s) {
             pack(next->value, s);
         }
     } else if (mp_obj_is_float(obj)) {
-        mp_float_t f = mp_obj_float_get(obj);
-        write_bytes(s, 1, 0xca);
-        write(s, &f, 4);
+        union Float { mp_float_t f;  uint32_t u; };
+        union Float data;
+        data.f = mp_obj_float_get(obj);
+        write1(s, 0xca);
+        write4(s, data.u);
     } else if (obj == mp_const_none) {
-        write_bytes(s, 1, 0xc0);
+        write1(s, 0xc0);
     } else if (obj == mp_const_false) {
-        write_bytes(s, 1, 0xc2);
+        write1(s, 0xc2);
     } else if (obj == mp_const_true) {
-        write_bytes(s, 1, 0xc3);
+        write1(s, 0xc3);
     } else {
         mp_raise_ValueError(translate("no packer"));
     }
@@ -252,7 +303,7 @@ void pack(mp_obj_t obj, msgpack_stream_t *s) {
 // unpacker
 
 mp_obj_t unpack(msgpack_stream_t *s) {
-    uint8_t code = read_bytes(s, 1);
+    uint8_t code = read1(s);
     if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) {
         // int
         return MP_OBJ_NEW_SMALL_INT((int8_t)code);
@@ -298,17 +349,24 @@ mp_obj_t unpack(msgpack_stream_t *s) {
             read(s, p, size);
             return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
         }
+        case 0xcc:
+            return MP_OBJ_NEW_SMALL_INT(read1(s));
+        case 0xcd:
+            return MP_OBJ_NEW_SMALL_INT(read2(s));
+        case 0xce:
+            return MP_OBJ_NEW_SMALL_INT(read4(s));
         case 0xca: {
-            float f;
-            read(s, &f, 4);
-            return mp_obj_new_float(f);
+            union Float { mp_float_t f;  uint32_t u; };
+            union Float data;
+            data.u = read4(s);
+            return mp_obj_new_float(data.f);
         }
         case 0xd0:
-            return MP_OBJ_NEW_SMALL_INT((int8_t)read_bytes(s, 1));
+            return MP_OBJ_NEW_SMALL_INT((int8_t)read1(s));
         case 0xd1:
-            return MP_OBJ_NEW_SMALL_INT((int16_t)read_bytes(s, 2));
+            return MP_OBJ_NEW_SMALL_INT((int16_t)read2(s));
         case 0xd2:
-            return MP_OBJ_NEW_SMALL_INT((int32_t)read_bytes(s, 4));
+            return MP_OBJ_NEW_SMALL_INT((int32_t)read4(s));
         case 0xd9:
         case 0xda:
         case 0xdb: {
@@ -345,9 +403,6 @@ mp_obj_t unpack(msgpack_stream_t *s) {
         case 0xc8:      // ext 16
         case 0xc9:      // ext 32
         case 0xcb:      // float 64
-        case 0xcc:      // uint 8
-        case 0xcd:      // uint 16
-        case 0xce:      // uint 32
         case 0xcf:      // uint 64
         case 0xd3:      // int 64
         case 0xd4:      // fixenxt 1

From b805bf8a6293ff4c6f8550946403e509a0aa9e37 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Thu, 12 Nov 2020 13:21:59 -0800
Subject: [PATCH 09/32] cleanup

---
 shared-module/msgpack/__init__.c | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index 505c1c9697..a825bb0f3b 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -67,13 +67,6 @@ STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
     }
 }
 
-/*
-STATIC uint32_t read_bytes(msgpack_stream_t *s, mp_uint_t n_bytes) {
-    uint32_t res = 0;
-    read(s, &res, n_bytes);
-    return res;
-}
-*/
 STATIC uint8_t read1(msgpack_stream_t *s) {
     uint8_t res = 0;
     read(s, &res, 1);
@@ -96,20 +89,6 @@ STATIC uint32_t read4(msgpack_stream_t *s) {
     return res;
 }
 
-/*
-STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
-    size_t n_bytes = 4;
-    switch (len_index) {
-        case 0:  n_bytes = 1;  break;
-        case 1:  n_bytes = 2;  break;
-        case 2:  n_bytes = 4;  break;
-    }
-    size_t res = 0;
-    read(s, &res, n_bytes);
-    return res;
-}
-*/
-
 STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
     size_t res;
     switch (len_index) {

From a310cde8c10878d83128c1a96b1dc4ecdf4b721a Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Thu, 12 Nov 2020 17:43:40 -0800
Subject: [PATCH 10/32] fixed res may not be initialized in read_size

---
 shared-module/msgpack/__init__.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index a825bb0f3b..3508755e0f 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -90,11 +90,12 @@ STATIC uint32_t read4(msgpack_stream_t *s) {
 }
 
 STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
-    size_t res;
+    size_t res = 0;
     switch (len_index) {
         case 0:  res = (size_t)read1(s);  break;
         case 1:  res = (size_t)read2(s);  break;
         case 2:  res = (size_t)read4(s);  break;
+        default: mp_raise_ValueError(translate("too big"));
     }
     return res;
 }

From 4b710796289756cb43275f19bafa9ff806e1011e Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Fri, 13 Nov 2020 17:00:32 -0800
Subject: [PATCH 11/32] update translations

---
 locale/circuitpython.pot | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot
index 51feb16c35..163d59428a 100644
--- a/locale/circuitpython.pot
+++ b/locale/circuitpython.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-12-01 18:38-0800\n"
+"POT-Creation-Date: 2020-12-01 18:39-0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -3528,6 +3528,10 @@ msgstr ""
 msgid "tobytes can be invoked for dense arrays only"
 msgstr ""
 
+#: shared-module/msgpack/__init__.c
+msgid "too big"
+msgstr ""
+
 #: shared-module/struct/__init__.c
 msgid "too many arguments provided with the given format"
 msgstr ""

From 59c3e25168fde52939948f5c1ffce13557bf4f4b Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Tue, 17 Nov 2020 11:56:36 -0800
Subject: [PATCH 12/32] disable on boards tight on memory. add stddef.h to
 imports (not actually needed).

---
 ports/atmel-samd/boards/bdmicro_vina_d21/mpconfigboard.mk       | 1 +
 .../boards/circuitplayground_express/mpconfigboard.mk           | 1 +
 .../boards/circuitplayground_express_crickit/mpconfigboard.mk   | 1 +
 .../boards/circuitplayground_express_displayio/mpconfigboard.mk | 1 +
 ports/atmel-samd/boards/feather_m0_express/mpconfigboard.mk     | 1 +
 .../boards/feather_m0_express_crickit/mpconfigboard.mk          | 1 +
 ports/atmel-samd/boards/feather_m0_supersized/mpconfigboard.mk  | 1 +
 ports/atmel-samd/boards/metro_m0_express/mpconfigboard.mk       | 1 +
 ports/atmel-samd/boards/snekboard/mpconfigboard.mk              | 1 +
 .../atmel-samd/boards/sparkfun_redboard_turbo/mpconfigboard.mk  | 1 +
 ports/cxd56/boards/spresense/mpconfigboard.mk                   | 2 ++
 shared-module/msgpack/__init__.c                                | 1 +
 12 files changed, 13 insertions(+)

diff --git a/ports/atmel-samd/boards/bdmicro_vina_d21/mpconfigboard.mk b/ports/atmel-samd/boards/bdmicro_vina_d21/mpconfigboard.mk
index a9885d064b..4e2314b8d5 100644
--- a/ports/atmel-samd/boards/bdmicro_vina_d21/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/bdmicro_vina_d21/mpconfigboard.mk
@@ -14,6 +14,7 @@ LONGINT_IMPL = MPZ
 CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_VECTORIO = 0
 
 CFLAGS_INLINE_LIMIT = 60
diff --git a/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk b/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk
index 13ec9e861c..505f5c145d 100644
--- a/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/circuitplayground_express/mpconfigboard.mk
@@ -15,6 +15,7 @@ LONGINT_IMPL = MPZ
 CIRCUITPY_DISPLAYIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 MICROPY_PY_ASYNC_AWAIT = 0
 
 SUPEROPT_GC = 0
diff --git a/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.mk b/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.mk
index 7aa45eb39e..c3be33134c 100644
--- a/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/circuitplayground_express_crickit/mpconfigboard.mk
@@ -16,6 +16,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_DISPLAYIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_PIXELBUF = 1
 CIRCUITPY_ROTARYIO = 0
 CIRCUITPY_RTC = 0
diff --git a/ports/atmel-samd/boards/circuitplayground_express_displayio/mpconfigboard.mk b/ports/atmel-samd/boards/circuitplayground_express_displayio/mpconfigboard.mk
index 3a43093a98..51e9b05af2 100644
--- a/ports/atmel-samd/boards/circuitplayground_express_displayio/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/circuitplayground_express_displayio/mpconfigboard.mk
@@ -16,6 +16,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_PIXELBUF = 0
 CIRCUITPY_ROTARYIO = 0
 CIRCUITPY_RTC = 0
diff --git a/ports/atmel-samd/boards/feather_m0_express/mpconfigboard.mk b/ports/atmel-samd/boards/feather_m0_express/mpconfigboard.mk
index dc02e1f60d..73ce47199b 100644
--- a/ports/atmel-samd/boards/feather_m0_express/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/feather_m0_express/mpconfigboard.mk
@@ -15,6 +15,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_VECTORIO = 0
 
 CFLAGS_INLINE_LIMIT = 60
diff --git a/ports/atmel-samd/boards/feather_m0_express_crickit/mpconfigboard.mk b/ports/atmel-samd/boards/feather_m0_express_crickit/mpconfigboard.mk
index 5624144e88..f06163b84a 100644
--- a/ports/atmel-samd/boards/feather_m0_express_crickit/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/feather_m0_express_crickit/mpconfigboard.mk
@@ -18,6 +18,7 @@ CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
 CIRCUITPY_GAMEPAD = 0
 CFLAGS_INLINE_LIMIT = 50
+CIRCUITPY_MSGPACK = 0
 
 # Include these Python libraries in firmware.
 FROZEN_MPY_DIRS += $(TOP)/frozen/Adafruit_CircuitPython_BusDevice
diff --git a/ports/atmel-samd/boards/feather_m0_supersized/mpconfigboard.mk b/ports/atmel-samd/boards/feather_m0_supersized/mpconfigboard.mk
index 6f7f2d8b67..0f1972abc3 100644
--- a/ports/atmel-samd/boards/feather_m0_supersized/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/feather_m0_supersized/mpconfigboard.mk
@@ -15,6 +15,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 # supersized, not ultra-supersized
 CIRCUITPY_VECTORIO = 0
 
diff --git a/ports/atmel-samd/boards/metro_m0_express/mpconfigboard.mk b/ports/atmel-samd/boards/metro_m0_express/mpconfigboard.mk
index 7dd9650003..0e9361a443 100644
--- a/ports/atmel-samd/boards/metro_m0_express/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/metro_m0_express/mpconfigboard.mk
@@ -14,6 +14,7 @@ LONGINT_IMPL = MPZ
 CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_VECTORIO = 0
 
 CFLAGS_INLINE_LIMIT = 60
diff --git a/ports/atmel-samd/boards/snekboard/mpconfigboard.mk b/ports/atmel-samd/boards/snekboard/mpconfigboard.mk
index e0262b6b22..d7540fe64d 100644
--- a/ports/atmel-samd/boards/snekboard/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/snekboard/mpconfigboard.mk
@@ -14,6 +14,7 @@ LONGINT_IMPL = MPZ
 CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_GAMEPAD = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_VECTORIO = 0
 
 CFLAGS_INLINE_LIMIT = 60
diff --git a/ports/atmel-samd/boards/sparkfun_redboard_turbo/mpconfigboard.mk b/ports/atmel-samd/boards/sparkfun_redboard_turbo/mpconfigboard.mk
index 5170f8a233..ddf7246dfb 100755
--- a/ports/atmel-samd/boards/sparkfun_redboard_turbo/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/sparkfun_redboard_turbo/mpconfigboard.mk
@@ -15,6 +15,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_GAMEPAD = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_VECTORIO = 0
 
 CFLAGS_INLINE_LIMIT = 60
diff --git a/ports/cxd56/boards/spresense/mpconfigboard.mk b/ports/cxd56/boards/spresense/mpconfigboard.mk
index 7b8ac6ff63..80c6e4b692 100644
--- a/ports/cxd56/boards/spresense/mpconfigboard.mk
+++ b/ports/cxd56/boards/spresense/mpconfigboard.mk
@@ -4,3 +4,5 @@ USB_PRODUCT = "Spresense"
 USB_MANUFACTURER = "Sony"
 
 INTERNAL_FLASH_FILESYSTEM = 1
+
+CIRCUITPY_MSGPACK = 0
diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index 3508755e0f..e550b01af9 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include <stddef.h>
 #include <stdio.h>
 #include <inttypes.h>
 

From 582a47d71a5d047abc7ee9a17b0940d1a1ed4b31 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Tue, 17 Nov 2020 16:26:09 -0800
Subject: [PATCH 13/32] rename read, write to read_bytes, write_bytes

---
 ports/cxd56/boards/spresense/mpconfigboard.mk |  2 --
 shared-module/msgpack/__init__.c              | 23 +++++++++----------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/ports/cxd56/boards/spresense/mpconfigboard.mk b/ports/cxd56/boards/spresense/mpconfigboard.mk
index 80c6e4b692..7b8ac6ff63 100644
--- a/ports/cxd56/boards/spresense/mpconfigboard.mk
+++ b/ports/cxd56/boards/spresense/mpconfigboard.mk
@@ -4,5 +4,3 @@ USB_PRODUCT = "Spresense"
 USB_MANUFACTURER = "Sony"
 
 INTERNAL_FLASH_FILESYSTEM = 1
-
-CIRCUITPY_MSGPACK = 0
diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index e550b01af9..5bc7d1f615 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -24,7 +24,6 @@
  * THE SOFTWARE.
  */
 
-#include <stddef.h>
 #include <stdio.h>
 #include <inttypes.h>
 
@@ -57,7 +56,7 @@ STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) {
 ////////////////////////////////////////////////////////////////
 // readers
 
-STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
+STATIC void read_bytes(msgpack_stream_t *s, void *buf, mp_uint_t size) {
     if (size == 0) return;
     mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode);
     if (s->errcode != 0) {
@@ -70,13 +69,13 @@ STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
 
 STATIC uint8_t read1(msgpack_stream_t *s) {
     uint8_t res = 0;
-    read(s, &res, 1);
+    read_bytes(s, &res, 1);
     return res;
 }
 
 STATIC uint16_t read2(msgpack_stream_t *s) {
     uint16_t res = 0;
-    read(s, &res, 2);
+    read_bytes(s, &res, 2);
     int n = 1;
     if (*(char *)&n == 1) res = __builtin_bswap16(res);
     return res;
@@ -84,7 +83,7 @@ STATIC uint16_t read2(msgpack_stream_t *s) {
 
 STATIC uint32_t read4(msgpack_stream_t *s) {
     uint32_t res = 0;
-    read(s, &res, 4);
+    read_bytes(s, &res, 4);
     int n = 1;
     if (*(char *)&n == 1) res = __builtin_bswap32(res);
     return res;
@@ -104,7 +103,7 @@ STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
 ////////////////////////////////////////////////////////////////
 // writers
 
-STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
+STATIC void write_bytes(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
     mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode);
     if (s->errcode != 0) {
         mp_raise_OSError(s->errcode);
@@ -115,19 +114,19 @@ STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
 }
 
 STATIC void write1(msgpack_stream_t *s, uint8_t obj) {
-    write(s, &obj, 1);
+    write_bytes(s, &obj, 1);
 }
 
 STATIC void write2(msgpack_stream_t *s, uint16_t obj) {
     int n = 1;
     if (*(char *)&n == 1) obj = __builtin_bswap16(obj);
-    write(s, &obj, 2);
+    write_bytes(s, &obj, 2);
 }
 
 STATIC void write4(msgpack_stream_t *s, uint32_t obj) {
     int n = 1;
     if (*(char *)&n == 1) obj = __builtin_bswap32(obj);
-    write(s, &obj, 4);
+    write_bytes(s, &obj, 4);
 }
 
 // compute and write msgpack size code (array structures)
@@ -294,7 +293,7 @@ mp_obj_t unpack(msgpack_stream_t *s) {
         size_t len = code & 0b11111;
         // allocate on stack; len < 32
         char str[len];
-        read(s, &str, len);
+        read_bytes(s, &str, len);
         return mp_obj_new_str(str, len);
     }
     if ((code & 0b11110000) == 0b10010000) {
@@ -327,7 +326,7 @@ mp_obj_t unpack(msgpack_stream_t *s) {
             vstr_t vstr;
             vstr_init_len(&vstr, size);
             byte *p = (byte*)vstr.buf;
-            read(s, p, size);
+            read_bytes(s, p, size);
             return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
         }
         case 0xcc:
@@ -356,7 +355,7 @@ mp_obj_t unpack(msgpack_stream_t *s) {
             vstr_t vstr;
             vstr_init_len(&vstr, size);
             byte *p = (byte*)vstr.buf;
-            read(s, p, size);
+            read_bytes(s, p, size);
             return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
         }
         case 0xde:

From 87d4184dd5e143494b77daa8d43bbc997f727095 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Wed, 18 Nov 2020 08:38:48 -0800
Subject: [PATCH 14/32] exclude spresense

---
 ports/cxd56/boards/spresense/mpconfigboard.mk | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ports/cxd56/boards/spresense/mpconfigboard.mk b/ports/cxd56/boards/spresense/mpconfigboard.mk
index 7b8ac6ff63..1e81c4683f 100644
--- a/ports/cxd56/boards/spresense/mpconfigboard.mk
+++ b/ports/cxd56/boards/spresense/mpconfigboard.mk
@@ -4,3 +4,5 @@ USB_PRODUCT = "Spresense"
 USB_MANUFACTURER = "Sony"
 
 INTERNAL_FLASH_FILESYSTEM = 1
+
+CIRCUITPY_MSGPACK = 0
\ No newline at end of file

From 546b15bf1abb66174c77eca4d21af71aa55384db Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@server.home>
Date: Wed, 18 Nov 2020 08:53:55 -0800
Subject: [PATCH 15/32] add trailing newline

---
 ports/cxd56/boards/spresense/mpconfigboard.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ports/cxd56/boards/spresense/mpconfigboard.mk b/ports/cxd56/boards/spresense/mpconfigboard.mk
index 1e81c4683f..80c6e4b692 100644
--- a/ports/cxd56/boards/spresense/mpconfigboard.mk
+++ b/ports/cxd56/boards/spresense/mpconfigboard.mk
@@ -5,4 +5,4 @@ USB_MANUFACTURER = "Sony"
 
 INTERNAL_FLASH_FILESYSTEM = 1
 
-CIRCUITPY_MSGPACK = 0
\ No newline at end of file
+CIRCUITPY_MSGPACK = 0

From 34bbcc4910014c3b4646d8005baf8244bac51f03 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Wed, 2 Dec 2020 11:52:29 -0800
Subject: [PATCH 16/32] exclude nrf/simmel due to lack of flash

---
 ports/nrf/boards/simmel/mpconfigboard.mk | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ports/nrf/boards/simmel/mpconfigboard.mk b/ports/nrf/boards/simmel/mpconfigboard.mk
index e34739c0f3..f702a336fd 100644
--- a/ports/nrf/boards/simmel/mpconfigboard.mk
+++ b/ports/nrf/boards/simmel/mpconfigboard.mk
@@ -16,6 +16,7 @@ CIRCUITPY_AUDIOMP3 = 0
 CIRCUITPY_BUSIO = 1
 CIRCUITPY_DISPLAYIO = 0
 CIRCUITPY_FRAMEBUFFERIO = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_NEOPIXEL_WRITE = 0
 CIRCUITPY_NVM = 0
 CIRCUITPY_PIXELBUF  = 0

From b5b6b6d0f2fe669e0304521d663b16388c63b15e Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Mon, 7 Dec 2020 15:16:16 -0800
Subject: [PATCH 17/32] add ExtType, update doc, add a test

---
 locale/circuitpython.pot           |  34 +++--
 shared-bindings/msgpack/ExtType.c  | 127 ++++++++++++++++++
 shared-bindings/msgpack/ExtType.h  |  40 ++++++
 shared-bindings/msgpack/__init__.c | 108 +++++++++++++---
 shared-bindings/msgpack/__init__.h |   2 +-
 shared-module/msgpack/__init__.c   | 199 +++++++++++++++++++----------
 shared-module/msgpack/__init__.h   |   4 +-
 tests/extmod/umsgpack_pack.py      |  30 +++++
 tests/extmod/umsgpack_pack.py.ext  |   5 +
 9 files changed, 447 insertions(+), 102 deletions(-)
 create mode 100644 shared-bindings/msgpack/ExtType.c
 create mode 100644 shared-bindings/msgpack/ExtType.h
 create mode 100644 tests/extmod/umsgpack_pack.py
 create mode 100644 tests/extmod/umsgpack_pack.py.ext

diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot
index e0b839ed3e..979f62466e 100644
--- a/locale/circuitpython.pot
+++ b/locale/circuitpython.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-12-01 18:39-0800\n"
+"POT-Creation-Date: 2020-12-07 15:35-0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -270,6 +270,10 @@ msgstr ""
 msgid "3-arg pow() not supported"
 msgstr ""
 
+#: shared-module/msgpack/__init__.c
+msgid "64 bit types"
+msgstr ""
+
 #: ports/atmel-samd/common-hal/countio/Counter.c
 #: ports/atmel-samd/common-hal/rotaryio/IncrementalEncoder.c
 msgid "A hardware interrupt channel is already in use"
@@ -2371,6 +2375,10 @@ msgstr ""
 msgid "circle can only be registered in one parent"
 msgstr ""
 
+#: shared-bindings/msgpack/ExtType.c
+msgid "code outside range 0~127"
+msgstr ""
+
 #: shared-bindings/displayio/Palette.c
 msgid "color buffer must be 3 bytes (RGB) or 4 bytes (RGB + pad byte)"
 msgstr ""
@@ -2451,6 +2459,10 @@ msgstr ""
 msgid "default 'except' must be last"
 msgstr ""
 
+#: shared-bindings/msgpack/__init__.c
+msgid "default is not a function"
+msgstr ""
+
 #: shared-bindings/audiobusio/PDMIn.c
 msgid ""
 "destination buffer must be a bytearray or array of type 'B' for bit_depth = 8"
@@ -2550,6 +2562,10 @@ msgstr ""
 msgid "expecting key:value for dict"
 msgstr ""
 
+#: shared-bindings/msgpack/__init__.c
+msgid "ext_hook is not a function"
+msgstr ""
+
 #: py/argcheck.c
 msgid "extra keyword arguments given"
 msgstr ""
@@ -3078,12 +3094,12 @@ msgstr ""
 msgid "no binding for nonlocal found"
 msgstr ""
 
-#: py/builtinimport.c
-msgid "no module named '%q'"
+#: shared-module/msgpack/__init__.c
+msgid "no default packer"
 msgstr ""
 
-#: shared-module/msgpack/__init__.c
-msgid "no packer"
+#: py/builtinimport.c
+msgid "no module named '%q'"
 msgstr ""
 
 #: shared-bindings/displayio/FourWire.c shared-bindings/displayio/I2CDisplay.c
@@ -3099,10 +3115,6 @@ msgstr ""
 msgid "no such attribute"
 msgstr ""
 
-#: shared-module/msgpack/__init__.c
-msgid "no unpacker found"
-msgstr ""
-
 #: ports/nrf/common-hal/_bleio/Connection.c
 msgid "non-UUID found in service_uuids_whitelist"
 msgstr ""
@@ -3560,10 +3572,6 @@ msgstr ""
 msgid "tobytes can be invoked for dense arrays only"
 msgstr ""
 
-#: shared-module/msgpack/__init__.c
-msgid "too big"
-msgstr ""
-
 #: shared-module/struct/__init__.c
 msgid "too many arguments provided with the given format"
 msgstr ""
diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
new file mode 100644
index 0000000000..3a4eee3884
--- /dev/null
+++ b/shared-bindings/msgpack/ExtType.c
@@ -0,0 +1,127 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2019  Bernhard Boser
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "py/runtime.h"
+#include "py/smallint.h"
+#include "py/objproperty.h"
+#include "shared-bindings/msgpack/ExtType.h"
+
+//| class ExtType:
+//|     """ExtType represents ext type in msgpack."""
+//|     def __init__(self, code: int, data: bytes) -> None:
+//|         """Constructor"""
+//|
+//|         :param int code: type code in range 0~127.
+//|         :param bytes data: representation.
+
+STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    mod_msgpack_extype_obj_t *self = m_new_obj(mod_msgpack_extype_obj_t);
+    self->base.type = &mod_msgpack_exttype_type;
+    enum { ARG_code, ARG_data };
+    static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_code, MP_ARG_INT | MP_ARG_REQUIRED },
+        { MP_QSTR_data, MP_ARG_OBJ | MP_ARG_REQUIRED },
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    int code = args[ARG_code].u_int;
+    if (code < 0 || code > 127) {
+        mp_raise_AttributeError(translate("code outside range 0~127"));
+    }
+    self->code = code;
+
+    mp_obj_t data = args[ARG_data].u_obj;
+    self->data = data;
+    return MP_OBJ_FROM_PTR(self);
+}
+
+
+//|     code: int
+//|     """The type code, in range 0~127."""
+//|
+
+STATIC mp_obj_t mod_msgpack_exttype_get_code(mp_obj_t self_in) {
+    mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    return MP_OBJ_NEW_SMALL_INT(self->code);
+}
+MP_DEFINE_CONST_FUN_OBJ_1(mod_msgpack_exttype_get_code_obj, mod_msgpack_exttype_get_code);
+
+STATIC mp_obj_t mod_msgpack_exttype_set_code(mp_obj_t self_in, mp_obj_t code_in) {
+    mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    int code = mp_obj_get_int(code_in);
+    if (code < 0 || code > 127) {
+        mp_raise_AttributeError(translate("code outside range 0~127"));
+    }
+    self->code = code;
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_exttype_set_code_obj, mod_msgpack_exttype_set_code);
+
+const mp_obj_property_t mod_msgpack_exttype_code_obj = {
+    .base.type = &mp_type_property,
+    .proxy = {(mp_obj_t)&mod_msgpack_exttype_get_code_obj,
+              (mp_obj_t)&mod_msgpack_exttype_set_code_obj,
+              (mp_obj_t)&mp_const_none_obj},
+};
+
+//|     data: bytes
+//|     """Data."""
+//|
+
+STATIC mp_obj_t mod_msgpack_exttype_get_data(mp_obj_t self_in) {
+    mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    return self->data;
+}
+MP_DEFINE_CONST_FUN_OBJ_1(mod_msgpack_exttype_get_data_obj, mod_msgpack_exttype_get_data);
+
+STATIC mp_obj_t mod_msgpack_exttype_set_data(mp_obj_t self_in, mp_obj_t data_in) {
+    mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    self->data = data_in;
+    return mp_const_none;
+}
+MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_exttype_set_data_obj, mod_msgpack_exttype_set_data);
+
+const mp_obj_property_t mod_msgpack_exttype_data_obj = {
+    .base.type = &mp_type_property,
+    .proxy = {(mp_obj_t)&mod_msgpack_exttype_get_data_obj,
+              (mp_obj_t)&mod_msgpack_exttype_set_data_obj,
+              (mp_obj_t)&mp_const_none_obj},
+};
+
+STATIC mp_rom_map_elem_t mod_msgpack_exttype_locals_dict_table[] = {
+    // Properties
+    { MP_ROM_QSTR(MP_QSTR_code), MP_ROM_PTR(&mod_msgpack_exttype_code_obj) },
+    { MP_ROM_QSTR(MP_QSTR_data), MP_ROM_PTR(&mod_msgpack_exttype_data_obj) },
+};
+STATIC MP_DEFINE_CONST_DICT(mod_msgpack_exttype_locals_dict, mod_msgpack_exttype_locals_dict_table);
+
+const mp_obj_type_t mod_msgpack_exttype_type = {
+    { &mp_type_type },
+    .name = MP_QSTR_ExtType,
+    .make_new = mod_msgpack_exttype_make_new,
+    .locals_dict = (mp_obj_dict_t*)&mod_msgpack_exttype_locals_dict,
+};
diff --git a/shared-bindings/msgpack/ExtType.h b/shared-bindings/msgpack/ExtType.h
new file mode 100644
index 0000000000..64173b2213
--- /dev/null
+++ b/shared-bindings/msgpack/ExtType.h
@@ -0,0 +1,40 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2020 Bernhard Boser
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK_EXTTYPE___INIT___H
+#define MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK_EXTTYPE___INIT___H
+
+#include "py/obj.h"
+
+typedef struct {
+    mp_obj_base_t base;
+    int32_t code;
+    mp_obj_t data;
+} mod_msgpack_extype_obj_t;
+
+extern const mp_obj_type_t mod_msgpack_exttype_type;
+
+#endif  // MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK_EXTTYPE___INIT___H
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index a222177f74..fcb86ac6c7 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -24,15 +24,24 @@
  * THE SOFTWARE.
  */
 
+#include <stdio.h>
+#include "py/obj.h"
+#include "py/runtime.h"
 #include "shared-bindings/msgpack/__init__.h"
 #include "shared-module/msgpack/__init__.h"
+#include "shared-bindings/msgpack/ExtType.h"
+
+#define MP_OBJ_IS_METH(o) (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type->name == MP_QSTR_bound_method))
 
 //| """Pack object in msgpack format
 //|
 //| The msgpack format is similar to json, except that the encoded data is binary.
-//| See https://msgpack.org for details.
+//| See https://msgpack.org for details. The module implements a subset of the cpython
+//| module msgpack-python.
 //|
-//| Example:
+//| Not implemented: 64-bit int, uint, float.
+//|
+//| Example 1:
 //|    import msgpack
 //|    from io import BytesIO
 //|
@@ -41,33 +50,98 @@
 //|    b.seek(0)
 //|    print(msgpack.unpack(b))"""
 //|
-
-//| def pack(obj: Any, buffer: WriteableBuffer) -> None:
-//|     """Pack obj to stream."""
-//|     ...
+//| Example 2: handling objects
 //|
+//|    from msgpack import pack, unpack, ExtType
+//|    from io import BytesIO
+//|
+//|    class MyClass:
+//|        def __init__(self, val):
+//|            self.value = val
+//|        def __str__(self):
+//|            return str(self.value)
+//|
+//|    data = MyClass(b'my_value')
+//|
+//|    def encoder(obj):
+//|        if isinstance(obj, MyClass):
+//|            return ExtType(1, obj.value)
+//|        return f"no encoder for {obj}"
+//|
+//|    def decoder(code, data):
+//|        if code == 1:
+//|            return MyClass(data)
+//|        return f"no decoder for type {code}"
+//|
+//|    buffer = BytesIO()
+//|    pack(data, buffer, default=encoder)
+//|    buffer.seek(0)
+//|    decoded = unpack(buffer, ext_hook=decoder)
+//|    print(f"{data} -> {buffer.getvalue()} -> {decoded}")
+//| """
 
-STATIC mp_obj_t mod_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) {
-    common_hal_msgpack_pack(obj, stream_obj);
+
+//| def pack(obj: object, buffer: WriteableBuffer, *, default: Function=None) -> None:
+//|     """Ouput object to buffer in msgpack format.
+//|     :param object obj: Object to convert to msgpack format.
+//|     :param ~_typing.WriteableBuffer buffer: buffer to write into
+//|     :param Optional[~_typing.Function] default:
+//|           function called for python objects that do not have
+//|           a representation in msgpack format.
+//|     """
+
+STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_obj, ARG_buffer, ARG_default };
+    STATIC const mp_arg_t allowed_args[] = {
+        { MP_QSTR_obj, MP_ARG_REQUIRED | MP_ARG_OBJ },
+        { MP_QSTR_buffer, MP_ARG_REQUIRED | MP_ARG_OBJ },
+        { MP_QSTR_default, MP_ARG_KW_ONLY | MP_ARG_OBJ, { .u_obj = mp_const_none } },
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
+
+    mp_obj_t handler = args[ARG_default].u_obj;
+    if (handler != mp_const_none && !MP_OBJ_IS_FUN(handler) && !MP_OBJ_IS_METH(handler)) {
+        mp_raise_ValueError(translate("default is not a function"));
+    }
+
+    common_hal_msgpack_pack(args[ARG_obj].u_obj, args[ARG_buffer].u_obj, handler);
     return mp_const_none;
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_2(mod_msgpack_pack_obj, mod_msgpack_pack);
+MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 
+//| def unpack(buffer: ReadableBuffer, *, ext_hook: Function=None, use_list: bool=True) -> object:
+//|     """Unpack and return one object from buffer.
+//|     :param ~_typing.ReadableBuffer buffer: buffer to read from
+//|     :param Optional[~_typing.Function] ext_hook: function called for objects in
+//|            msgpack ext format.
+//|     :param Optional[bool] use_list: return array as list or tuple (use_list=False).
+//|     :return object: object read from buffer.
+//|     """
 
-//| def unpack(buffer: ReadableBuffer) -> Any:
-//|     """Unpack and return one object (in msgpack format) from stream.
-//|     Call repeatedly to read multiple objects from the stream."""
-//|     ...
-//|
+STATIC mp_obj_t mod_msgpack_unpack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
+    enum { ARG_buffer, ARG_ext_hook, ARG_use_list };
+    STATIC const mp_arg_t allowed_args[] = {
+        { MP_QSTR_buffer, MP_ARG_REQUIRED | MP_ARG_OBJ, },
+        { MP_QSTR_ext_hook, MP_ARG_KW_ONLY | MP_ARG_OBJ, { .u_obj = mp_const_none } },
+        { MP_QSTR_use_list, MP_ARG_KW_ONLY | MP_ARG_BOOL, { .u_bool = true } },
+    };
+    mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
+    mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
-STATIC mp_obj_t mod_msgpack_unpack(mp_obj_t stream_obj) {
-    return common_hal_msgpack_unpack(stream_obj);
+    mp_obj_t hook = args[ARG_ext_hook].u_obj;
+    if (hook != mp_const_none && !MP_OBJ_IS_FUN(hook) && !MP_OBJ_IS_METH(hook)) {
+        mp_raise_ValueError(translate("ext_hook is not a function"));
+    }
+
+    return common_hal_msgpack_unpack(args[ARG_buffer].u_obj, hook, args[ARG_use_list].u_bool);
 }
-STATIC MP_DEFINE_CONST_FUN_OBJ_1(mod_msgpack_unpack_obj, mod_msgpack_unpack);
+MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_unpack_obj, 1, mod_msgpack_unpack);
 
 
 STATIC const mp_rom_map_elem_t msgpack_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_msgpack) },
+    { MP_ROM_QSTR(MP_QSTR_ExtType), MP_ROM_PTR(&mod_msgpack_exttype_type) },
     { MP_ROM_QSTR(MP_QSTR_pack), MP_ROM_PTR(&mod_msgpack_pack_obj) },
     { MP_ROM_QSTR(MP_QSTR_unpack), MP_ROM_PTR(&mod_msgpack_unpack_obj) },
 };
diff --git a/shared-bindings/msgpack/__init__.h b/shared-bindings/msgpack/__init__.h
index 4836cc62e7..a02ead0bd0 100644
--- a/shared-bindings/msgpack/__init__.h
+++ b/shared-bindings/msgpack/__init__.h
@@ -29,6 +29,6 @@
 
 #include "py/obj.h"
 
-// Nothing now.
+// nothing for now
 
 #endif  // MICROPY_INCLUDED_SHARED_BINDINGS_MSGPACK___INIT___H
diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index 5bc7d1f615..204312109a 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -27,6 +27,7 @@
 #include <stdio.h>
 #include <inttypes.h>
 
+#include "py/obj.h"
 #include "py/binary.h"
 #include "py/objarray.h"
 #include "py/objlist.h"
@@ -36,6 +37,7 @@
 #include "py/stream.h"
 
 #include "supervisor/shared/translate.h"
+#include "shared-bindings/msgpack/ExtType.h"
 
 ////////////////////////////////////////////////////////////////
 // stream management
@@ -56,7 +58,7 @@ STATIC msgpack_stream_t get_stream(mp_obj_t stream_obj, int flags) {
 ////////////////////////////////////////////////////////////////
 // readers
 
-STATIC void read_bytes(msgpack_stream_t *s, void *buf, mp_uint_t size) {
+STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
     if (size == 0) return;
     mp_uint_t ret = s->read(s->stream_obj, buf, size, &s->errcode);
     if (s->errcode != 0) {
@@ -69,13 +71,13 @@ STATIC void read_bytes(msgpack_stream_t *s, void *buf, mp_uint_t size) {
 
 STATIC uint8_t read1(msgpack_stream_t *s) {
     uint8_t res = 0;
-    read_bytes(s, &res, 1);
+    read(s, &res, 1);
     return res;
 }
 
 STATIC uint16_t read2(msgpack_stream_t *s) {
     uint16_t res = 0;
-    read_bytes(s, &res, 2);
+    read(s, &res, 2);
     int n = 1;
     if (*(char *)&n == 1) res = __builtin_bswap16(res);
     return res;
@@ -83,19 +85,18 @@ STATIC uint16_t read2(msgpack_stream_t *s) {
 
 STATIC uint32_t read4(msgpack_stream_t *s) {
     uint32_t res = 0;
-    read_bytes(s, &res, 4);
+    read(s, &res, 4);
     int n = 1;
     if (*(char *)&n == 1) res = __builtin_bswap32(res);
     return res;
 }
 
 STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
-    size_t res = 0;
+    size_t res;
     switch (len_index) {
         case 0:  res = (size_t)read1(s);  break;
         case 1:  res = (size_t)read2(s);  break;
         case 2:  res = (size_t)read4(s);  break;
-        default: mp_raise_ValueError(translate("too big"));
     }
     return res;
 }
@@ -103,7 +104,7 @@ STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
 ////////////////////////////////////////////////////////////////
 // writers
 
-STATIC void write_bytes(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
+STATIC void write(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
     mp_uint_t ret = s->write(s->stream_obj, buf, size, &s->errcode);
     if (s->errcode != 0) {
         mp_raise_OSError(s->errcode);
@@ -114,19 +115,19 @@ STATIC void write_bytes(msgpack_stream_t *s, const void *buf, mp_uint_t size) {
 }
 
 STATIC void write1(msgpack_stream_t *s, uint8_t obj) {
-    write_bytes(s, &obj, 1);
+    write(s, &obj, 1);
 }
 
 STATIC void write2(msgpack_stream_t *s, uint16_t obj) {
     int n = 1;
     if (*(char *)&n == 1) obj = __builtin_bswap16(obj);
-    write_bytes(s, &obj, 2);
+    write(s, &obj, 2);
 }
 
 STATIC void write4(msgpack_stream_t *s, uint32_t obj) {
     int n = 1;
     if (*(char *)&n == 1) obj = __builtin_bswap32(obj);
-    write_bytes(s, &obj, 4);
+    write(s, &obj, 4);
 }
 
 // compute and write msgpack size code (array structures)
@@ -178,15 +179,34 @@ STATIC void pack_int(msgpack_stream_t *s, int32_t x) {
     }
 }
 
-void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
+STATIC void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
     write_size(s, 0xc4, len);
     for (size_t i=0;  i<len;  i++) {
         write1(s, data[i]);
     }
 }
 
-void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
-    // size_t len = strlen(str);
+STATIC void  pack_ext(msgpack_stream_t *s, int8_t code, const uint8_t* data, size_t len) {
+    if (len == 1) {
+        write1(s, 0xd4);
+    } else if (len == 2) {
+        write1(s, 0xd5);
+    } else if (len == 4) {
+        write1(s, 0xd6);
+    } else if (len == 8) {
+        write1(s, 0xd7);
+    } else if (len == 16) {
+        write1(s, 0xd8);
+    } else {
+        write_size(s, 0xc7, len);
+    }
+    write1(s, code);    // type byte
+    for (size_t i=0;  i<len;  i++) {
+        write1(s, data[i]);
+    }
+}
+
+STATIC void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
     if (len < 32) {
         write1(s, 0b10100000 | (uint8_t)len);
     } else {
@@ -197,7 +217,7 @@ void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
     }
 }
 
-void pack_array(msgpack_stream_t *s, size_t len) {
+STATIC void pack_array(msgpack_stream_t *s, size_t len) {
     // only writes the header, manually write the objects after calling pack_array!
     if (len < 16) {
         write1(s, 0b10010000 | (uint8_t)len);
@@ -210,7 +230,7 @@ void pack_array(msgpack_stream_t *s, size_t len) {
     }
 }
 
-void pack_dict(msgpack_stream_t *s, size_t len) {
+STATIC void pack_dict(msgpack_stream_t *s, size_t len) {
     // only writes the header, manually write the objects after calling pack_array!
     if (len < 16) {
         write1(s, 0b10000000 | (uint8_t)len);
@@ -223,7 +243,7 @@ void pack_dict(msgpack_stream_t *s, size_t len) {
     }
 }
 
-void pack(mp_obj_t obj, msgpack_stream_t *s) {
+STATIC void pack(mp_obj_t obj, msgpack_stream_t *s, mp_obj_t default_handler) {
     if (MP_OBJ_IS_SMALL_INT(obj)) {
         // int
         int32_t x = MP_OBJ_SMALL_INT_VALUE(obj);
@@ -233,6 +253,11 @@ void pack(mp_obj_t obj, msgpack_stream_t *s) {
         size_t len;
         const char *data = mp_obj_str_get_data(obj, &len);
         pack_str(s, data, len);
+    } else if (MP_OBJ_IS_TYPE(obj, &mod_msgpack_exttype_type)) {
+        mod_msgpack_extype_obj_t *ext = MP_OBJ_TO_PTR(obj);
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer_raise(ext->data, &bufinfo, MP_BUFFER_READ);
+        pack_ext(s, ext->code, bufinfo.buf, bufinfo.len);
     } else if (MP_OBJ_IS_TYPE(obj, &mp_type_bytes)) {
         // bytes
         mp_buffer_info_t bufinfo;
@@ -243,14 +268,14 @@ void pack(mp_obj_t obj, msgpack_stream_t *s) {
         mp_obj_tuple_t *self = MP_OBJ_TO_PTR(obj);
         pack_array(s, self->len);
         for (size_t i=0;  i<self->len;  i++) {
-            pack(self->items[i], s);
+            pack(self->items[i], s, default_handler);
         }
     } else if (MP_OBJ_IS_TYPE(obj, &mp_type_list)) {
         // list (layout differs from tuple)
         mp_obj_list_t *self = MP_OBJ_TO_PTR(obj);
         pack_array(s, self->len);
         for (size_t i=0;  i<self->len;  i++) {
-            pack(self->items[i], s);
+            pack(self->items[i], s, default_handler);
         }
     } else if (MP_OBJ_IS_TYPE(obj, &mp_type_dict)) {
         // dict
@@ -259,8 +284,8 @@ void pack(mp_obj_t obj, msgpack_stream_t *s) {
         size_t cur = 0;
         mp_map_elem_t *next = NULL;
         while ((next = dict_iter_next(self, &cur)) != NULL) {
-            pack(next->key, s);
-            pack(next->value, s);
+            pack(next->key, s, default_handler);
+            pack(next->value, s, default_handler);
         }
     } else if (mp_obj_is_float(obj)) {
         union Float { mp_float_t f;  uint32_t u; };
@@ -275,14 +300,60 @@ void pack(mp_obj_t obj, msgpack_stream_t *s) {
     } else if (obj == mp_const_true) {
         write1(s, 0xc3);
     } else {
-        mp_raise_ValueError(translate("no packer"));
+        if (default_handler != mp_const_none) {
+            // set default_handler to mp_const_none to avoid infinite recursion
+            // this also precludes some valid outputs
+            pack(mp_call_function_1(default_handler, obj), s, mp_const_none);
+        } else {
+            mp_raise_ValueError(translate("no default packer"));
+        }
     }
 }
 
 ////////////////////////////////////////////////////////////////
 // unpacker
 
-mp_obj_t unpack(msgpack_stream_t *s) {
+STATIC mp_obj_t unpack(msgpack_stream_t *s, mp_obj_t ext_hook, bool use_list);
+
+STATIC mp_obj_t unpack_array_elements(msgpack_stream_t *s, size_t size, mp_obj_t ext_hook, bool use_list) {
+    if (use_list) {
+        mp_obj_list_t *t = MP_OBJ_TO_PTR(mp_obj_new_list(size, NULL));
+        for (size_t i=0; i<size; i++) {
+            t->items[i] = unpack(s, ext_hook, use_list);
+        }
+        return MP_OBJ_FROM_PTR(t);
+    } else {
+        mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(size, NULL));
+        for (size_t i=0; i<size; i++) {
+            t->items[i] = unpack(s, ext_hook, use_list);
+        }
+        return MP_OBJ_FROM_PTR(t);
+    }
+}
+
+STATIC mp_obj_t unpack_bytes(msgpack_stream_t *s, size_t size) {
+    vstr_t vstr;
+    vstr_init_len(&vstr, size);
+    byte *p = (byte*)vstr.buf;
+    read(s, p, size);
+    return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+}
+
+STATIC mp_obj_t unpack_ext(msgpack_stream_t *s, size_t size, mp_obj_t ext_hook) {
+    int8_t code = read1(s);
+    mp_obj_t data = unpack_bytes(s, size);
+    if (ext_hook != mp_const_none) {
+        return mp_call_function_2(ext_hook, MP_OBJ_NEW_SMALL_INT(code), data);
+    } else {
+        mod_msgpack_extype_obj_t *o = m_new_obj(mod_msgpack_extype_obj_t);
+        o->base.type = &mod_msgpack_exttype_type;
+        o->code = code;
+        o->data = data;
+        return MP_OBJ_FROM_PTR(o);
+    }
+}
+
+STATIC mp_obj_t unpack(msgpack_stream_t *s, mp_obj_t ext_hook, bool use_list) {
     uint8_t code = read1(s);
     if (((code & 0b10000000) == 0) || ((code & 0b11100000) == 0b11100000)) {
         // int
@@ -293,24 +364,19 @@ mp_obj_t unpack(msgpack_stream_t *s) {
         size_t len = code & 0b11111;
         // allocate on stack; len < 32
         char str[len];
-        read_bytes(s, &str, len);
+        read(s, &str, len);
         return mp_obj_new_str(str, len);
     }
     if ((code & 0b11110000) == 0b10010000) {
-        // array (tuple)
-        size_t len = code & 0b1111;
-        mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
-        for (size_t i=0; i<len; i++) {
-            t->items[i] = unpack(s);
-        }
-        return MP_OBJ_FROM_PTR(t);
+        // array (list / tuple)
+        return unpack_array_elements(s, code & 0b1111, ext_hook, use_list);
     }
     if ((code & 0b11110000) == 0b10000000) {
         // map (dict)
         size_t len = code & 0b1111;
         mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
         for (size_t i=0; i<len; i++) {
-            mp_obj_dict_store(d, unpack(s), unpack(s));
+            mp_obj_dict_store(d, unpack(s, ext_hook, use_list), unpack(s, ext_hook, use_list));
         }
         return MP_OBJ_FROM_PTR(d);
     }
@@ -322,31 +388,23 @@ mp_obj_t unpack(msgpack_stream_t *s) {
         case 0xc5:
         case 0xc6: {
             // bin 8, 16, 32
-            size_t size = read_size(s, code-0xc4);
-            vstr_t vstr;
-            vstr_init_len(&vstr, size);
-            byte *p = (byte*)vstr.buf;
-            read_bytes(s, p, size);
-            return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
+            return unpack_bytes(s, read_size(s, code-0xc4));
         }
-        case 0xcc:
-            return MP_OBJ_NEW_SMALL_INT(read1(s));
-        case 0xcd:
-            return MP_OBJ_NEW_SMALL_INT(read2(s));
-        case 0xce:
-            return MP_OBJ_NEW_SMALL_INT(read4(s));
+        case 0xcc: // uint8
+        case 0xd0: // int8
+            return MP_OBJ_NEW_SMALL_INT((int8_t)read1(s));
+        case 0xcd: // uint16
+        case 0xd1: // int16
+            return MP_OBJ_NEW_SMALL_INT((int16_t)read2(s));
+        case 0xce: // uint32
+        case 0xd2: // int32
+            return MP_OBJ_NEW_SMALL_INT((int32_t)read4(s));
         case 0xca: {
             union Float { mp_float_t f;  uint32_t u; };
             union Float data;
             data.u = read4(s);
             return mp_obj_new_float(data.f);
         }
-        case 0xd0:
-            return MP_OBJ_NEW_SMALL_INT((int8_t)read1(s));
-        case 0xd1:
-            return MP_OBJ_NEW_SMALL_INT((int16_t)read2(s));
-        case 0xd2:
-            return MP_OBJ_NEW_SMALL_INT((int32_t)read4(s));
         case 0xd9:
         case 0xda:
         case 0xdb: {
@@ -355,7 +413,7 @@ mp_obj_t unpack(msgpack_stream_t *s) {
             vstr_t vstr;
             vstr_init_len(&vstr, size);
             byte *p = (byte*)vstr.buf;
-            read_bytes(s, p, size);
+            read(s, p, size);
             return mp_obj_new_str_from_vstr(&mp_type_str, &vstr);
         }
         case 0xde:
@@ -364,43 +422,46 @@ mp_obj_t unpack(msgpack_stream_t *s) {
             size_t len = read_size(s, code - 0xde + 1);
             mp_obj_dict_t *d = MP_OBJ_TO_PTR(mp_obj_new_dict(len));
             for (size_t i=0; i<len; i++) {
-                mp_obj_dict_store(d, unpack(s), unpack(s));
+                mp_obj_dict_store(d, unpack(s, ext_hook, use_list), unpack(s, ext_hook, use_list));
             }
             return MP_OBJ_FROM_PTR(d);
         }
         case 0xdc:
         case 0xdd: {
             // array 16 & 32
-            size_t len = read_size(s, code - 0xdc + 1);
-            mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(len, NULL));
-            for (size_t i=0; i<len; i++) {
-                t->items[i] = unpack(s);
-            }
-            return MP_OBJ_FROM_PTR(t);
+            size_t size = read_size(s, code - 0xdc + 1);
+            return unpack_array_elements(s, size, ext_hook, use_list);
         }
-        case 0xc1:      // never used
+        case 0xd4:      // fixenxt 1
+            return unpack_ext(s, 1, ext_hook);
+        case 0xd5:      // fixenxt 2
+            return unpack_ext(s, 2, ext_hook);
+        case 0xd6:      // fixenxt 4
+            return unpack_ext(s, 4, ext_hook);
+        case 0xd7:      // fixenxt 8
+            return unpack_ext(s, 8, ext_hook);
+        case 0xd8:      // fixenxt 16
+            return unpack_ext(s, 16, ext_hook);
         case 0xc7:      // ext 8
         case 0xc8:      // ext 16
-        case 0xc9:      // ext 32
+        case 0xc9:
+            // ext 8, 16, 32
+            return unpack_ext(s, read_size(s, code-0xc7), ext_hook);
+        case 0xc1:      // never used
         case 0xcb:      // float 64
         case 0xcf:      // uint 64
         case 0xd3:      // int 64
-        case 0xd4:      // fixenxt 1
-        case 0xd5:      // fixenxt 2
-        case 0xd6:      // fixenxt 4
-        case 0xd7:      // fixenxt 8
-        case 0xd8:      // fixenxt 16
         default:
-            mp_raise_ValueError(translate("no unpacker found"));
+            mp_raise_NotImplementedError(translate("64 bit types"));
     }
 }
 
-void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj) {
+void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj, mp_obj_t default_handler) {
     msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE);
-    pack(obj, &stream);
+    pack(obj, &stream, default_handler);
 }
 
-mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj) {
+mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj, mp_obj_t ext_hook, bool use_list) {
     msgpack_stream_t stream = get_stream(stream_obj, MP_STREAM_OP_WRITE);
-    return unpack(&stream);
+    return unpack(&stream, ext_hook, use_list);
 }
diff --git a/shared-module/msgpack/__init__.h b/shared-module/msgpack/__init__.h
index 0a5e7852ad..88b4809f95 100644
--- a/shared-module/msgpack/__init__.h
+++ b/shared-module/msgpack/__init__.h
@@ -28,7 +28,7 @@
 
 #include "py/stream.h"
 
-void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj);
-mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj);
+void common_hal_msgpack_pack(mp_obj_t obj, mp_obj_t stream_obj, mp_obj_t default_handler);
+mp_obj_t common_hal_msgpack_unpack(mp_obj_t stream_obj, mp_obj_t ext_hook, bool use_list);
 
 #endif
diff --git a/tests/extmod/umsgpack_pack.py b/tests/extmod/umsgpack_pack.py
new file mode 100644
index 0000000000..d9f0005e1b
--- /dev/null
+++ b/tests/extmod/umsgpack_pack.py
@@ -0,0 +1,30 @@
+try:
+    from uio import BytesIO
+    import umsgpack as msgpack
+except:
+    try:
+        from io import BytesIO
+        import msgpack
+    except ImportError:
+        print("SKIP")
+        raise SystemExit
+
+b = BytesIO()
+msgpack.pack(False, s)
+print(b.getvalue())
+
+b = BytesIO()
+msgpack.pack({"a": (-1, 0, 2, [3, None], 128)}, b)
+print(b.getvalue())
+
+# pack to a small-int not allowed
+try:
+    msgpack.pack(123, 1)
+except (AttributeError, OSError): # CPython and uPy have different errors
+    print('Exception')
+
+# pack to an object not allowed
+try:
+    msgpack.pack(123, {})
+except (AttributeError, OSError): # CPython and uPy have different errors
+    print('Exception')
diff --git a/tests/extmod/umsgpack_pack.py.ext b/tests/extmod/umsgpack_pack.py.ext
new file mode 100644
index 0000000000..2f966be069
--- /dev/null
+++ b/tests/extmod/umsgpack_pack.py.ext
@@ -0,0 +1,5 @@
+b'\xc2'
+b'\x82\xa1a\x96\xff\x00\x02\x92\x03\xc0\xd1\x00\x80\xc4\x07abcdefg\xa1b\xd4\x05x'
+Exception ExtType
+Exception to int
+Exception to object

From 534b48fcfe688d88455bd8421fa02255ecc8508b Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Mon, 7 Dec 2020 16:08:16 -0800
Subject: [PATCH 18/32] remove a ~ from doc that causes an error; add ExtType.c
 to circuitpy_defns.mk

---
 locale/circuitpython.pot          | 2 +-
 py/circuitpy_defns.mk             | 1 +
 shared-bindings/msgpack/ExtType.c | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot
index 979f62466e..7acfbe1ded 100644
--- a/locale/circuitpython.pot
+++ b/locale/circuitpython.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-12-07 15:35-0800\n"
+"POT-Creation-Date: 2020-12-07 16:05-0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
diff --git a/py/circuitpy_defns.mk b/py/circuitpy_defns.mk
index f13f69bcf2..39ff2e380f 100644
--- a/py/circuitpy_defns.mk
+++ b/py/circuitpy_defns.mk
@@ -407,6 +407,7 @@ $(filter $(SRC_PATTERNS), \
 	microcontroller/ResetReason.c \
 	microcontroller/RunMode.c \
 	msgpack/__init__.c \
+	msgpack/ExtType.c \
 	supervisor/RunReason.c \
 )
 
diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
index 3a4eee3884..dfccbd3ef2 100644
--- a/shared-bindings/msgpack/ExtType.c
+++ b/shared-bindings/msgpack/ExtType.c
@@ -34,7 +34,7 @@
 //|     def __init__(self, code: int, data: bytes) -> None:
 //|         """Constructor"""
 //|
-//|         :param int code: type code in range 0~127.
+//|         :param int code: type code in range 0 .. 127.
 //|         :param bytes data: representation.
 
 STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {

From 9412c3f58e9340d9300cd048f566feda7c48c53c Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Mon, 7 Dec 2020 16:22:09 -0800
Subject: [PATCH 19/32] moved misplaced triple-quote

---
 shared-bindings/msgpack/ExtType.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
index dfccbd3ef2..daac55dc44 100644
--- a/shared-bindings/msgpack/ExtType.c
+++ b/shared-bindings/msgpack/ExtType.c
@@ -32,10 +32,9 @@
 //| class ExtType:
 //|     """ExtType represents ext type in msgpack."""
 //|     def __init__(self, code: int, data: bytes) -> None:
-//|         """Constructor"""
-//|
-//|         :param int code: type code in range 0 .. 127.
-//|         :param bytes data: representation.
+//|         """Constructor
+//|         :param int code: type code in range 0~127.
+//|         :param bytes data: representation."""
 
 STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     mod_msgpack_extype_obj_t *self = m_new_obj(mod_msgpack_extype_obj_t);

From 5875a27fa8f09dc27c3d25c13f4aa28c22c0279b Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Mon, 7 Dec 2020 16:32:38 -0800
Subject: [PATCH 20/32] removed extra triple quote

---
 shared-bindings/msgpack/__init__.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index fcb86ac6c7..5f1fac1375 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -48,7 +48,7 @@
 //|    b = BytesIO()
 //|    msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, b)
 //|    b.seek(0)
-//|    print(msgpack.unpack(b))"""
+//|    print(msgpack.unpack(b))
 //|
 //| Example 2: handling objects
 //|

From 9903c9e855f07b6660f6a89546030d47cd9c70e6 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Mon, 7 Dec 2020 16:50:52 -0800
Subject: [PATCH 21/32] fixed (?) typespec for callback functions

---
 shared-bindings/msgpack/__init__.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 5f1fac1375..08d535c58a 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -85,7 +85,7 @@
 //|     """Ouput object to buffer in msgpack format.
 //|     :param object obj: Object to convert to msgpack format.
 //|     :param ~_typing.WriteableBuffer buffer: buffer to write into
-//|     :param Optional[~_typing.Function] default:
+//|     :param Optional[~_typing.Callable[[object], None]] default:
 //|           function called for python objects that do not have
 //|           a representation in msgpack format.
 //|     """
@@ -113,7 +113,7 @@ MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 //| def unpack(buffer: ReadableBuffer, *, ext_hook: Function=None, use_list: bool=True) -> object:
 //|     """Unpack and return one object from buffer.
 //|     :param ~_typing.ReadableBuffer buffer: buffer to read from
-//|     :param Optional[~_typing.Function] ext_hook: function called for objects in
+//|     :param Optional[~_typing.Callable[[int, bytes], object]] ext_hook: function called for objects in
 //|            msgpack ext format.
 //|     :param Optional[bool] use_list: return array as list or tuple (use_list=False).
 //|     :return object: object read from buffer.

From 413885a529db041413a1e1b259389afce3602d7e Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Mon, 7 Dec 2020 17:45:21 -0800
Subject: [PATCH 22/32] fix typspec in function declarations

---
 shared-bindings/msgpack/__init__.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 08d535c58a..0660ee78b9 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -81,7 +81,7 @@
 //| """
 
 
-//| def pack(obj: object, buffer: WriteableBuffer, *, default: Function=None) -> None:
+//| def pack(obj: object, buffer: WriteableBuffer, *, default: Callable[[object], None]]=None) -> None:
 //|     """Ouput object to buffer in msgpack format.
 //|     :param object obj: Object to convert to msgpack format.
 //|     :param ~_typing.WriteableBuffer buffer: buffer to write into
@@ -110,7 +110,7 @@ STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map
 }
 MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 
-//| def unpack(buffer: ReadableBuffer, *, ext_hook: Function=None, use_list: bool=True) -> object:
+//| def unpack(buffer: ReadableBuffer, *, ext_hook: Callable[[int, bytes], object]]=None, use_list: bool=True) -> object:
 //|     """Unpack and return one object from buffer.
 //|     :param ~_typing.ReadableBuffer buffer: buffer to read from
 //|     :param Optional[~_typing.Callable[[int, bytes], object]] ext_hook: function called for objects in

From 1f14d393645d2f3939b8e378148b440ed237e2ce Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Tue, 8 Dec 2020 11:50:45 -0800
Subject: [PATCH 23/32] remove two extra closing angular brackets

---
 shared-bindings/msgpack/__init__.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 0660ee78b9..ade88b10dd 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -81,7 +81,7 @@
 //| """
 
 
-//| def pack(obj: object, buffer: WriteableBuffer, *, default: Callable[[object], None]]=None) -> None:
+//| def pack(obj: object, buffer: WriteableBuffer, *, default: Callable[[object], None]=None) -> None:
 //|     """Ouput object to buffer in msgpack format.
 //|     :param object obj: Object to convert to msgpack format.
 //|     :param ~_typing.WriteableBuffer buffer: buffer to write into
@@ -110,7 +110,7 @@ STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map
 }
 MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 
-//| def unpack(buffer: ReadableBuffer, *, ext_hook: Callable[[int, bytes], object]]=None, use_list: bool=True) -> object:
+//| def unpack(buffer: ReadableBuffer, *, ext_hook: Callable[[int, bytes], object]=None, use_list: bool=True) -> object:
 //|     """Unpack and return one object from buffer.
 //|     :param ~_typing.ReadableBuffer buffer: buffer to read from
 //|     :param Optional[~_typing.Callable[[int, bytes], object]] ext_hook: function called for objects in

From 9e7a7b9dc422e3031c914a90f9ceda9b21ed27f8 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Tue, 8 Dec 2020 14:38:20 -0800
Subject: [PATCH 24/32] uglify typing hints

---
 shared-bindings/msgpack/__init__.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index ade88b10dd..59ff694bb8 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -81,7 +81,7 @@
 //| """
 
 
-//| def pack(obj: object, buffer: WriteableBuffer, *, default: Callable[[object], None]=None) -> None:
+//| def pack(obj: object, buffer: WriteableBuffer, *, default: Union[Callable[[object], None],None]=None) -> None:
 //|     """Ouput object to buffer in msgpack format.
 //|     :param object obj: Object to convert to msgpack format.
 //|     :param ~_typing.WriteableBuffer buffer: buffer to write into
@@ -110,7 +110,7 @@ STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map
 }
 MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 
-//| def unpack(buffer: ReadableBuffer, *, ext_hook: Callable[[int, bytes], object]=None, use_list: bool=True) -> object:
+//| def unpack(buffer: ReadableBuffer, *, ext_hook: Union[Callable[[int, bytes], object],None],None]=None, use_list: bool=True) -> object:
 //|     """Unpack and return one object from buffer.
 //|     :param ~_typing.ReadableBuffer buffer: buffer to read from
 //|     :param Optional[~_typing.Callable[[int, bytes], object]] ext_hook: function called for objects in

From a5c3dc286c38c3c007513db58c09a88308c2a863 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Tue, 8 Dec 2020 15:48:56 -0800
Subject: [PATCH 25/32] typing ...

---
 shared-bindings/msgpack/__init__.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 59ff694bb8..ea1e430877 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -80,8 +80,7 @@
 //|    print(f"{data} -> {buffer.getvalue()} -> {decoded}")
 //| """
 
-
-//| def pack(obj: object, buffer: WriteableBuffer, *, default: Union[Callable[[object], None],None]=None) -> None:
+//| def pack(obj: object, buffer: WriteableBuffer, *, default: Union[Callable[[object], None], None] = None) -> None:
 //|     """Ouput object to buffer in msgpack format.
 //|     :param object obj: Object to convert to msgpack format.
 //|     :param ~_typing.WriteableBuffer buffer: buffer to write into
@@ -110,7 +109,8 @@ STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map
 }
 MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 
-//| def unpack(buffer: ReadableBuffer, *, ext_hook: Union[Callable[[int, bytes], object],None],None]=None, use_list: bool=True) -> object:
+
+//| def unpack(buffer: ReadableBuffer, *, ext_hook: Union[Callable[[int, bytes], object], None] = None, use_list: bool=True) -> object:
 //|     """Unpack and return one object from buffer.
 //|     :param ~_typing.ReadableBuffer buffer: buffer to read from
 //|     :param Optional[~_typing.Callable[[int, bytes], object]] ext_hook: function called for objects in

From 3c724321d863c79dce0ec4d11e12fd6c9fa27c4c Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Wed, 9 Dec 2020 12:01:09 -0800
Subject: [PATCH 26/32] cosmetics

---
 shared-bindings/msgpack/ExtType.c  | 2 --
 shared-bindings/msgpack/__init__.c | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
index daac55dc44..272c1ccebc 100644
--- a/shared-bindings/msgpack/ExtType.c
+++ b/shared-bindings/msgpack/ExtType.c
@@ -61,7 +61,6 @@ STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n
 
 //|     code: int
 //|     """The type code, in range 0~127."""
-//|
 
 STATIC mp_obj_t mod_msgpack_exttype_get_code(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
@@ -89,7 +88,6 @@ const mp_obj_property_t mod_msgpack_exttype_code_obj = {
 
 //|     data: bytes
 //|     """Data."""
-//|
 
 STATIC mp_obj_t mod_msgpack_exttype_get_data(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index ea1e430877..06c3900b5d 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -46,7 +46,7 @@
 //|    from io import BytesIO
 //|
 //|    b = BytesIO()
-//|    msgpack.pack({'list': [True, False, None, 1, 'abc'], 'str': 'blah'}, b)
+//|    msgpack.pack({'list': [True, False, None, 1, 3.14], 'str': 'blah'}, b)
 //|    b.seek(0)
 //|    print(msgpack.unpack(b))
 //|

From 6d2329fb09ce47eaa5cdf0dbf9c5e1e1ef37632a Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Thu, 10 Dec 2020 10:59:26 -0800
Subject: [PATCH 27/32] add blank lines after all comment sections

---
 shared-bindings/msgpack/ExtType.c  | 9 ++++++---
 shared-bindings/msgpack/__init__.c | 8 ++++++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
index 272c1ccebc..6f0a16b999 100644
--- a/shared-bindings/msgpack/ExtType.c
+++ b/shared-bindings/msgpack/ExtType.c
@@ -35,7 +35,8 @@
 //|         """Constructor
 //|         :param int code: type code in range 0~127.
 //|         :param bytes data: representation."""
-
+//|     ...
+//|
 STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     mod_msgpack_extype_obj_t *self = m_new_obj(mod_msgpack_extype_obj_t);
     self->base.type = &mod_msgpack_exttype_type;
@@ -61,7 +62,8 @@ STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n
 
 //|     code: int
 //|     """The type code, in range 0~127."""
-
+//|     ...
+//|
 STATIC mp_obj_t mod_msgpack_exttype_get_code(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
     return MP_OBJ_NEW_SMALL_INT(self->code);
@@ -88,7 +90,8 @@ const mp_obj_property_t mod_msgpack_exttype_code_obj = {
 
 //|     data: bytes
 //|     """Data."""
-
+//|     ...
+//|
 STATIC mp_obj_t mod_msgpack_exttype_get_data(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
     return self->data;
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 06c3900b5d..b41b32d45f 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -79,6 +79,8 @@
 //|    decoded = unpack(buffer, ext_hook=decoder)
 //|    print(f"{data} -> {buffer.getvalue()} -> {decoded}")
 //| """
+//|     ...
+//|
 
 //| def pack(obj: object, buffer: WriteableBuffer, *, default: Union[Callable[[object], None], None] = None) -> None:
 //|     """Ouput object to buffer in msgpack format.
@@ -88,7 +90,8 @@
 //|           function called for python objects that do not have
 //|           a representation in msgpack format.
 //|     """
-
+//|     ...
+//|
 STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_obj, ARG_buffer, ARG_default };
     STATIC const mp_arg_t allowed_args[] = {
@@ -118,7 +121,8 @@ MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 //|     :param Optional[bool] use_list: return array as list or tuple (use_list=False).
 //|     :return object: object read from buffer.
 //|     """
-
+//|     ...
+//|
 STATIC mp_obj_t mod_msgpack_unpack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_buffer, ARG_ext_hook, ARG_use_list };
     STATIC const mp_arg_t allowed_args[] = {

From d29184b5a0fb5c54663d48a8e8906564c3d518ad Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Thu, 10 Dec 2020 12:15:43 -0800
Subject: [PATCH 28/32] try without ... in docs

---
 shared-bindings/msgpack/ExtType.c  | 3 ---
 shared-bindings/msgpack/__init__.c | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
index 6f0a16b999..5dc3d4992e 100644
--- a/shared-bindings/msgpack/ExtType.c
+++ b/shared-bindings/msgpack/ExtType.c
@@ -35,7 +35,6 @@
 //|         """Constructor
 //|         :param int code: type code in range 0~127.
 //|         :param bytes data: representation."""
-//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     mod_msgpack_extype_obj_t *self = m_new_obj(mod_msgpack_extype_obj_t);
@@ -62,7 +61,6 @@ STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n
 
 //|     code: int
 //|     """The type code, in range 0~127."""
-//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_exttype_get_code(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
@@ -90,7 +88,6 @@ const mp_obj_property_t mod_msgpack_exttype_code_obj = {
 
 //|     data: bytes
 //|     """Data."""
-//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_exttype_get_data(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index b41b32d45f..5dc45dde2c 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -79,7 +79,6 @@
 //|    decoded = unpack(buffer, ext_hook=decoder)
 //|    print(f"{data} -> {buffer.getvalue()} -> {decoded}")
 //| """
-//|     ...
 //|
 
 //| def pack(obj: object, buffer: WriteableBuffer, *, default: Union[Callable[[object], None], None] = None) -> None:
@@ -90,7 +89,6 @@
 //|           function called for python objects that do not have
 //|           a representation in msgpack format.
 //|     """
-//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_obj, ARG_buffer, ARG_default };
@@ -121,7 +119,6 @@ MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 //|     :param Optional[bool] use_list: return array as list or tuple (use_list=False).
 //|     :return object: object read from buffer.
 //|     """
-//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_unpack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_buffer, ARG_ext_hook, ARG_use_list };

From c875d7c22d447b9a29bcef62af7b46c902684d72 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Sat, 19 Dec 2020 19:06:43 -0800
Subject: [PATCH 29/32] speedup pack_bin, ext, str; catch short reads

---
 shared-module/msgpack/__init__.c | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/shared-module/msgpack/__init__.c b/shared-module/msgpack/__init__.c
index 204312109a..39178d46e1 100644
--- a/shared-module/msgpack/__init__.c
+++ b/shared-module/msgpack/__init__.c
@@ -67,6 +67,9 @@ STATIC void read(msgpack_stream_t *s, void *buf, mp_uint_t size) {
     if (ret == 0) {
         mp_raise_msg(&mp_type_EOFError, NULL);
     }
+    if (ret < size) {
+        mp_raise_ValueError(translate("short read"));
+    }
 }
 
 STATIC uint8_t read1(msgpack_stream_t *s) {
@@ -92,7 +95,7 @@ STATIC uint32_t read4(msgpack_stream_t *s) {
 }
 
 STATIC size_t read_size(msgpack_stream_t *s, uint8_t len_index) {
-    size_t res;
+    size_t res = 0;
     switch (len_index) {
         case 0:  res = (size_t)read1(s);  break;
         case 1:  res = (size_t)read2(s);  break;
@@ -181,9 +184,7 @@ STATIC void pack_int(msgpack_stream_t *s, int32_t x) {
 
 STATIC void pack_bin(msgpack_stream_t *s, const uint8_t* data, size_t len) {
     write_size(s, 0xc4, len);
-    for (size_t i=0;  i<len;  i++) {
-        write1(s, data[i]);
-    }
+    if (len > 0) write(s, data, len);
 }
 
 STATIC void  pack_ext(msgpack_stream_t *s, int8_t code, const uint8_t* data, size_t len) {
@@ -201,9 +202,7 @@ STATIC void  pack_ext(msgpack_stream_t *s, int8_t code, const uint8_t* data, siz
         write_size(s, 0xc7, len);
     }
     write1(s, code);    // type byte
-    for (size_t i=0;  i<len;  i++) {
-        write1(s, data[i]);
-    }
+    if (len > 0) write(s, data, len);
 }
 
 STATIC void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
@@ -212,9 +211,7 @@ STATIC void pack_str(msgpack_stream_t *s, const char* str, size_t len) {
     } else {
         write_size(s, 0xd9, len);
     }
-    for (size_t l=0;  l<len;  l++) {
-        write1(s, str[l]);
-    }
+    if (len > 0) write(s, str, len);
 }
 
 STATIC void pack_array(msgpack_stream_t *s, size_t len) {

From 90a299bb1e6c80971b576dc51dd50bca6220b6f7 Mon Sep 17 00:00:00 2001
From: Bernhard Boser <boser@berkeley.edu>
Date: Tue, 5 Jan 2021 11:17:09 -0800
Subject: [PATCH 30/32] added //| to (hopefully) get CI to pass

---
 locale/circuitpython.pot           | 6 +++++-
 shared-bindings/msgpack/ExtType.c  | 2 ++
 shared-bindings/msgpack/__init__.c | 2 ++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/locale/circuitpython.pot b/locale/circuitpython.pot
index cea26c6bac..578257e2d0 100644
--- a/locale/circuitpython.pot
+++ b/locale/circuitpython.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2020-12-08 09:56-0800\n"
+"POT-Creation-Date: 2021-01-05 11:08-0800\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -3438,6 +3438,10 @@ msgstr ""
 msgid "shape must be a tuple"
 msgstr ""
 
+#: shared-module/msgpack/__init__.c
+msgid "short read"
+msgstr ""
+
 #: py/objstr.c
 msgid "sign not allowed in string format specifier"
 msgstr ""
diff --git a/shared-bindings/msgpack/ExtType.c b/shared-bindings/msgpack/ExtType.c
index 5dc3d4992e..e9ddd32bce 100644
--- a/shared-bindings/msgpack/ExtType.c
+++ b/shared-bindings/msgpack/ExtType.c
@@ -61,6 +61,7 @@ STATIC mp_obj_t mod_msgpack_exttype_make_new(const mp_obj_type_t *type, size_t n
 
 //|     code: int
 //|     """The type code, in range 0~127."""
+//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_exttype_get_code(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
@@ -88,6 +89,7 @@ const mp_obj_property_t mod_msgpack_exttype_code_obj = {
 
 //|     data: bytes
 //|     """Data."""
+//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_exttype_get_data(mp_obj_t self_in) {
     mod_msgpack_extype_obj_t *self = MP_OBJ_TO_PTR(self_in);
diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index 5dc45dde2c..e1b6b8fce6 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -89,6 +89,7 @@
 //|           function called for python objects that do not have
 //|           a representation in msgpack format.
 //|     """
+//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_pack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_obj, ARG_buffer, ARG_default };
@@ -119,6 +120,7 @@ MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 //|     :param Optional[bool] use_list: return array as list or tuple (use_list=False).
 //|     :return object: object read from buffer.
 //|     """
+//|     ...
 //|
 STATIC mp_obj_t mod_msgpack_unpack(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) {
     enum { ARG_buffer, ARG_ext_hook, ARG_use_list };

From cdad59fbc2d681d06a21aefedb00a6b6946cce1c Mon Sep 17 00:00:00 2001
From: Scott Shawcroft <scott@tannewt.org>
Date: Wed, 6 Jan 2021 09:03:58 -0800
Subject: [PATCH 31/32] Fix doc build with extra blank lines

---
 shared-bindings/msgpack/__init__.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/shared-bindings/msgpack/__init__.c b/shared-bindings/msgpack/__init__.c
index e1b6b8fce6..f5c47bb218 100644
--- a/shared-bindings/msgpack/__init__.c
+++ b/shared-bindings/msgpack/__init__.c
@@ -41,7 +41,8 @@
 //|
 //| Not implemented: 64-bit int, uint, float.
 //|
-//| Example 1:
+//| Example 1::
+//|
 //|    import msgpack
 //|    from io import BytesIO
 //|
@@ -50,7 +51,7 @@
 //|    b.seek(0)
 //|    print(msgpack.unpack(b))
 //|
-//| Example 2: handling objects
+//| Example 2: handling objects::
 //|
 //|    from msgpack import pack, unpack, ExtType
 //|    from io import BytesIO
@@ -78,11 +79,13 @@
 //|    buffer.seek(0)
 //|    decoded = unpack(buffer, ext_hook=decoder)
 //|    print(f"{data} -> {buffer.getvalue()} -> {decoded}")
+//|
 //| """
 //|
 
 //| def pack(obj: object, buffer: WriteableBuffer, *, default: Union[Callable[[object], None], None] = None) -> None:
 //|     """Ouput object to buffer in msgpack format.
+//|
 //|     :param object obj: Object to convert to msgpack format.
 //|     :param ~_typing.WriteableBuffer buffer: buffer to write into
 //|     :param Optional[~_typing.Callable[[object], None]] default:
@@ -114,10 +117,12 @@ MP_DEFINE_CONST_FUN_OBJ_KW(mod_msgpack_pack_obj, 1, mod_msgpack_pack);
 
 //| def unpack(buffer: ReadableBuffer, *, ext_hook: Union[Callable[[int, bytes], object], None] = None, use_list: bool=True) -> object:
 //|     """Unpack and return one object from buffer.
+//|
 //|     :param ~_typing.ReadableBuffer buffer: buffer to read from
 //|     :param Optional[~_typing.Callable[[int, bytes], object]] ext_hook: function called for objects in
 //|            msgpack ext format.
 //|     :param Optional[bool] use_list: return array as list or tuple (use_list=False).
+//|
 //|     :return object: object read from buffer.
 //|     """
 //|     ...

From 13efbf24e58b2be940cad44e4f0816d6daa67202 Mon Sep 17 00:00:00 2001
From: foamyguy <foamyguy@gmail.com>
Date: Sat, 9 Jan 2021 10:33:56 -0600
Subject: [PATCH 32/32] disable msgpack on boards without room

---
 ports/atmel-samd/boards/qtpy_m0_haxpress/mpconfigboard.mk   | 1 +
 ports/atmel-samd/boards/stackrduino_m0_pro/mpconfigboard.mk | 1 +
 ports/nrf/boards/pca10100/mpconfigboard.mk                  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/ports/atmel-samd/boards/qtpy_m0_haxpress/mpconfigboard.mk b/ports/atmel-samd/boards/qtpy_m0_haxpress/mpconfigboard.mk
index a63f142742..8773c5771d 100644
--- a/ports/atmel-samd/boards/qtpy_m0_haxpress/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/qtpy_m0_haxpress/mpconfigboard.mk
@@ -17,6 +17,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 
 SUPEROPT_GC = 0
 
diff --git a/ports/atmel-samd/boards/stackrduino_m0_pro/mpconfigboard.mk b/ports/atmel-samd/boards/stackrduino_m0_pro/mpconfigboard.mk
index 704d265141..734be2d145 100644
--- a/ports/atmel-samd/boards/stackrduino_m0_pro/mpconfigboard.mk
+++ b/ports/atmel-samd/boards/stackrduino_m0_pro/mpconfigboard.mk
@@ -16,6 +16,7 @@ CIRCUITPY_BITBANGIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_COUNTIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_VECTORIO = 0
 CIRCUITPY_BUSDEVICE = 0
 
diff --git a/ports/nrf/boards/pca10100/mpconfigboard.mk b/ports/nrf/boards/pca10100/mpconfigboard.mk
index 86ba654548..76d15e6081 100644
--- a/ports/nrf/boards/pca10100/mpconfigboard.mk
+++ b/ports/nrf/boards/pca10100/mpconfigboard.mk
@@ -15,6 +15,7 @@ CIRCUITPY_DISPLAYIO = 0
 CIRCUITPY_FRAMEBUFFERIO = 0
 CIRCUITPY_FREQUENCYIO = 0
 CIRCUITPY_I2CPERIPHERAL = 0
+CIRCUITPY_MSGPACK = 0
 CIRCUITPY_NEOPIXEL_WRITE = 0
 CIRCUITPY_NVM = 0
 CIRCUITPY_PIXELBUF  = 0