Make natmods work again.

And put back our magic number, because our bytecode format differs
from upstream

drop btree & framebuf natmods, they had additional problems I didn't
want to fix right now.

.github/workflows/run-tests.yml

@@ -55,8 +55,6 @@ jobs:
       run: |
         make -C examples/natmod/features1
         make -C examples/natmod/features2
-        make -C examples/natmod/btree
-        make -C examples/natmod/framebuf
         make -C examples/natmod/uheapq
         make -C examples/natmod/urandom
         make -C examples/natmod/ure

docs/reference/mpyfiles.rst

@@ -66,7 +66,7 @@ If importing an .mpy file fails then try the following:
     print()
 
 * Check the validity of the .mpy file by inspecting the first two bytes of
-  the file.  The first byte should be an uppercase 'M' and the second byte
+  the file.  The first byte should be an uppercase 'C' and the second byte
   will be the version number, which should match the system version from above.
   If it doesn't match then rebuild the .mpy file.
 
@@ -144,7 +144,7 @@ The .mpy header is:
 ======  ================================
 size    field
 ======  ================================
-byte    value 0x4d (ASCII 'M')
+byte    value 0x43 (ASCII 'C')
 byte    .mpy version number
 byte    feature flags
 byte    number of bits in a small int

examples/natmod/.gitignore

@@ -0,0 +1 @@
+*.mpy

examples/natmod/features0/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features0
+
+# Source files (.c or .py)
+SRC = features0.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features0/features0.c

@@ -0,0 +1,40 @@
+/* This example demonstrates the following features in a native module:
+    - defining a simple function exposed to Python
+    - defining a local, helper C function
+    - getting and creating integer objects
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// Helper function to compute factorial
+STATIC mp_int_t factorial_helper(mp_int_t x) {
+    if (x == 0) {
+        return 1;
+    }
+    return x * factorial_helper(x - 1);
+}
+
+// This is the function which will be called from Python, as factorial(x)
+STATIC mp_obj_t factorial(mp_obj_t x_obj) {
+    // Extract the integer from the MicroPython input object
+    mp_int_t x = mp_obj_get_int(x_obj);
+    // Calculate the factorial
+    mp_int_t result = factorial_helper(x);
+    // Convert the result to a MicroPython integer object and return it
+    return mp_obj_new_int(result);
+}
+// Define a Python reference to the function above
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_obj, factorial);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Make the function available in the module's namespace
+    mp_store_global(MP_QSTR_factorial, MP_OBJ_FROM_PTR(&factorial_obj));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/features1/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features1
+
+# Source files (.c or .py)
+SRC = features1.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features1/features1.c

@@ -0,0 +1,106 @@
+/* This example demonstrates the following features in a native module:
+    - defining simple functions exposed to Python
+    - defining local, helper C functions
+    - defining constant integers and strings exposed to Python
+    - getting and creating integer objects
+    - creating Python lists
+    - raising exceptions
+    - allocating memory
+    - BSS and constant data (rodata)
+    - relocated pointers in rodata
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// BSS (zero) data
+uint16_t data16[4];
+
+// Constant data (rodata)
+const uint8_t table8[] = { 0, 1, 1, 2, 3, 5, 8, 13 };
+const uint16_t table16[] = { 0x1000, 0x2000 };
+
+// Constant data pointing to BSS/constant data
+uint16_t *const table_ptr16a[] = { &data16[0], &data16[1], &data16[2], &data16[3] };
+const uint16_t *const table_ptr16b[] = { &table16[0], &table16[1] };
+
+// A simple function that adds its 2 arguments (must be integers)
+STATIC mp_obj_t add(mp_obj_t x_in, mp_obj_t y_in) {
+    mp_int_t x = mp_obj_get_int(x_in);
+    mp_int_t y = mp_obj_get_int(y_in);
+    return mp_obj_new_int(x + y);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add);
+
+// A local helper function (not exposed to Python)
+STATIC mp_int_t fibonacci_helper(mp_int_t x) {
+    if (x < MP_ARRAY_SIZE(table8)) {
+        return table8[x];
+    } else {
+        return fibonacci_helper(x - 1) + fibonacci_helper(x - 2);
+    }
+}
+
+// A function which computes Fibonacci numbers
+STATIC mp_obj_t fibonacci(mp_obj_t x_in) {
+    mp_int_t x = mp_obj_get_int(x_in);
+    if (x < 0) {
+        mp_raise_ValueError(MP_ERROR_TEXT("can't compute negative Fibonacci number"));
+    }
+    return mp_obj_new_int(fibonacci_helper(x));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(fibonacci_obj, fibonacci);
+
+// A function that accesses the BSS data
+STATIC mp_obj_t access(size_t n_args, const mp_obj_t *args) {
+    if (n_args == 0) {
+        // Create a list holding all items from data16
+        mp_obj_list_t *lst = MP_OBJ_TO_PTR(mp_obj_new_list(MP_ARRAY_SIZE(data16), NULL));
+        for (int i = 0; i < MP_ARRAY_SIZE(data16); ++i) {
+            lst->items[i] = mp_obj_new_int(data16[i]);
+        }
+        return MP_OBJ_FROM_PTR(lst);
+    } else if (n_args == 1) {
+        // Get one item from data16
+        mp_int_t idx = mp_obj_get_int(args[0]) & 3;
+        return mp_obj_new_int(data16[idx]);
+    } else {
+        // Set one item in data16 (via table_ptr16a)
+        mp_int_t idx = mp_obj_get_int(args[0]) & 3;
+        *table_ptr16a[idx] = mp_obj_get_int(args[1]);
+        return mp_const_none;
+    }
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(access_obj, 0, 2, access);
+
+// A function that allocates memory and creates a bytearray
+STATIC mp_obj_t make_array(void) {
+    uint16_t *ptr = m_new(uint16_t, MP_ARRAY_SIZE(table_ptr16b));
+    for (int i = 0; i < MP_ARRAY_SIZE(table_ptr16b); ++i) {
+        ptr[i] = *table_ptr16b[i];
+    }
+    return mp_obj_new_bytearray_by_ref(sizeof(uint16_t) * MP_ARRAY_SIZE(table_ptr16b), ptr);
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(make_array_obj, make_array);
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Messages can be printed as usually
+    mp_printf(&mp_plat_print, "initialising module self=%p\n", self);
+
+    // Make the functions available in the module's namespace
+    mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj));
+    mp_store_global(MP_QSTR_fibonacci, MP_OBJ_FROM_PTR(&fibonacci_obj));
+    mp_store_global(MP_QSTR_access, MP_OBJ_FROM_PTR(&access_obj));
+    mp_store_global(MP_QSTR_make_array, MP_OBJ_FROM_PTR(&make_array_obj));
+
+    // Add some constants to the module's namespace
+    mp_store_global(MP_QSTR_VAL, MP_OBJ_NEW_SMALL_INT(42));
+    mp_store_global(MP_QSTR_MSG, MP_OBJ_NEW_QSTR(MP_QSTR_HELLO_MICROPYTHON));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/features2/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features2
+
+# Source files (.c or .py)
+SRC = main.c prod.c test.py
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features2/main.c

@@ -0,0 +1,83 @@
+/* This example demonstrates the following features in a native module:
+    - using floats
+    - defining additional code in Python (see test.py)
+    - have extra C code in a separate file (see prod.c)
+*/
+
+// Include the header file to get access to the MicroPython API
+#include "py/dynruntime.h"
+
+// Include the header for auxiliary C code for this module
+#include "prod.h"
+
+// Automatically detect if this module should include double-precision code.
+// If double precision is supported by the target architecture then it can
+// be used in native module regardless of what float setting the target
+// MicroPython runtime uses (being none, float or double).
+#if defined(__i386__) || defined(__x86_64__) || (defined(__ARM_FP) && (__ARM_FP & 8))
+#define USE_DOUBLE 1
+#else
+#define USE_DOUBLE 0
+#endif
+
+// A function that uses the default float type configured for the current target
+// This default can be overridden by specifying MICROPY_FLOAT_IMPL at the make level
+STATIC mp_obj_t add(mp_obj_t x, mp_obj_t y) {
+    return mp_obj_new_float(mp_obj_get_float(x) + mp_obj_get_float(y));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add);
+
+// A function that explicitly uses single precision floats
+STATIC mp_obj_t add_f(mp_obj_t x, mp_obj_t y) {
+    return mp_obj_new_float_from_f(mp_obj_get_float_to_f(x) + mp_obj_get_float_to_f(y));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_f_obj, add_f);
+
+#if USE_DOUBLE
+// A function that explicitly uses double precision floats
+STATIC mp_obj_t add_d(mp_obj_t x, mp_obj_t y) {
+    return mp_obj_new_float_from_d(mp_obj_get_float_to_d(x) + mp_obj_get_float_to_d(y));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_d_obj, add_d);
+#endif
+
+// A function that computes the product of floats in an array.
+// This function uses the most general C argument interface, which is more difficult
+// to use but has access to the globals dict of the module via self->globals.
+STATIC mp_obj_t productf(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // Check number of arguments is valid
+    mp_arg_check_num(n_args, n_kw, 1, 1, false);
+
+    // Extract buffer pointer and verify typecode
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_RW);
+    if (bufinfo.typecode != 'f') {
+        mp_raise_ValueError(MP_ERROR_TEXT("expecting float array"));
+    }
+
+    // Compute product, store result back in first element of array
+    float *ptr = bufinfo.buf;
+    float prod = prod_array(bufinfo.len / sizeof(*ptr), ptr);
+    ptr[0] = prod;
+
+    return mp_const_none;
+}
+
+// This is the entry point and is called when the module is imported
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Make the functions available in the module's namespace
+    mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj));
+    mp_store_global(MP_QSTR_add_f, MP_OBJ_FROM_PTR(&add_f_obj));
+    #if USE_DOUBLE
+    mp_store_global(MP_QSTR_add_d, MP_OBJ_FROM_PTR(&add_d_obj));
+    #endif
+
+    // The productf function uses the most general C argument interface
+    mp_store_global(MP_QSTR_productf, MP_DYNRUNTIME_MAKE_FUNCTION(productf));
+
+    // This must be last, it restores the globals dict
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/features2/prod.c

@@ -0,0 +1,9 @@
+#include "prod.h"
+
+float prod_array(int n, float *ar) {
+    float ans = 1;
+    for (int i = 0; i < n; ++i) {
+        ans *= ar[i];
+    }
+    return ans;
+}

examples/natmod/features2/prod.h

@@ -0,0 +1 @@
+float prod_array(int n, float *ar);

examples/natmod/features2/test.py

@@ -0,0 +1,29 @@
+# This Python code will be merged with the C code in main.c
+
+import array
+
+
+def isclose(a, b):
+    return abs(a - b) < 1e-3
+
+
+def test():
+    tests = [
+        isclose(add(0.1, 0.2), 0.3),
+        isclose(add_f(0.1, 0.2), 0.3),
+    ]
+
+    ar = array.array("f", [1, 2, 3.5])
+    productf(ar)
+    tests.append(isclose(ar[0], 7))
+
+    if "add_d" in globals():
+        tests.append(isclose(add_d(0.1, 0.2), 0.3))
+
+    print(tests)
+
+    if not all(tests):
+        raise SystemExit(1)
+
+
+test()

examples/natmod/features3/Makefile

@@ -0,0 +1,14 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module
+MOD = features3
+
+# Source files (.c or .py)
+SRC = features3.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+# Include to get the rules for compiling and linking the module
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/features3/features3.c

@@ -0,0 +1,60 @@
+/* This example demonstrates the following features in a native module:
+    - using types
+    - using constant objects
+    - creating dictionaries
+*/
+
+// Include the header file to get access to the MicroPython API.
+#include "py/dynruntime.h"
+
+// A function that returns a tuple of object types.
+STATIC mp_obj_t get_types(void) {
+    return mp_obj_new_tuple(9, ((mp_obj_t []) {
+        MP_OBJ_FROM_PTR(&mp_type_type),
+        MP_OBJ_FROM_PTR(&mp_type_NoneType),
+        MP_OBJ_FROM_PTR(&mp_type_bool),
+        MP_OBJ_FROM_PTR(&mp_type_int),
+        MP_OBJ_FROM_PTR(&mp_type_str),
+        MP_OBJ_FROM_PTR(&mp_type_bytes),
+        MP_OBJ_FROM_PTR(&mp_type_tuple),
+        MP_OBJ_FROM_PTR(&mp_type_list),
+        MP_OBJ_FROM_PTR(&mp_type_dict),
+    }));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(get_types_obj, get_types);
+
+// A function that returns a tuple of constant objects.
+STATIC mp_obj_t get_const_objects(void) {
+    return mp_obj_new_tuple(5, ((mp_obj_t []) {
+        mp_const_none,
+        mp_const_false,
+        mp_const_true,
+        mp_const_empty_bytes,
+        mp_const_empty_tuple,
+    }));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(get_const_objects_obj, get_const_objects);
+
+// A function that creates a dictionary from the given arguments.
+STATIC mp_obj_t make_dict(size_t n_args, const mp_obj_t *args) {
+    mp_obj_t dict = mp_obj_new_dict(n_args / 2);
+    for (; n_args >= 2; n_args -= 2, args += 2) {
+        mp_obj_dict_store(dict, args[0], args[1]);
+    }
+    return dict;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(make_dict_obj, 0, MP_OBJ_FUN_ARGS_MAX, make_dict);
+
+// This is the entry point and is called when the module is imported.
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    // This must be first, it sets up the globals dict and other things.
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    // Make the functions available in the module's namespace.
+    mp_store_global(MP_QSTR_make_dict, MP_OBJ_FROM_PTR(&make_dict_obj));
+    mp_store_global(MP_QSTR_get_types, MP_OBJ_FROM_PTR(&get_types_obj));
+    mp_store_global(MP_QSTR_get_const_objects, MP_OBJ_FROM_PTR(&get_const_objects_obj));
+
+    // This must be last, it restores the globals dict.
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/uheapq/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in uheapq so it can coexist)
+MOD = uheapq_$(ARCH)
+
+# Source files (.c or .py)
+SRC = uheapq.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/uheapq/uheapq.c

@@ -0,0 +1,16 @@
+#define MICROPY_PY_UHEAPQ (1)
+
+#include "py/dynruntime.h"
+
+#include "extmod/moduheapq.c"
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uheapq));
+    mp_store_global(MP_QSTR_heappush, MP_OBJ_FROM_PTR(&mod_uheapq_heappush_obj));
+    mp_store_global(MP_QSTR_heappop, MP_OBJ_FROM_PTR(&mod_uheapq_heappop_obj));
+    mp_store_global(MP_QSTR_heapify, MP_OBJ_FROM_PTR(&mod_uheapq_heapify_obj));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/urandom/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in urandom so it can coexist)
+MOD = urandom_$(ARCH)
+
+# Source files (.c or .py)
+SRC = urandom.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/urandom/urandom.c

@@ -0,0 +1,33 @@
+#define MICROPY_PY_URANDOM (1)
+#define MICROPY_PY_URANDOM_EXTRA_FUNCS (1)
+
+#include "py/dynruntime.h"
+
+// Dynamic native modules don't support a data section so these must go in the BSS
+uint32_t yasmarang_pad, yasmarang_n, yasmarang_d;
+uint8_t yasmarang_dat;
+
+#include "extmod/modurandom.c"
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    yasmarang_pad = 0xeda4baba;
+    yasmarang_n = 69;
+    yasmarang_d = 233;
+
+    mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_urandom));
+    mp_store_global(MP_QSTR_getrandbits, MP_OBJ_FROM_PTR(&mod_urandom_getrandbits_obj));
+    mp_store_global(MP_QSTR_seed, MP_OBJ_FROM_PTR(&mod_urandom_seed_obj));
+    #if MICROPY_PY_URANDOM_EXTRA_FUNCS
+    mp_store_global(MP_QSTR_randrange, MP_OBJ_FROM_PTR(&mod_urandom_randrange_obj));
+    mp_store_global(MP_QSTR_randint, MP_OBJ_FROM_PTR(&mod_urandom_randint_obj));
+    mp_store_global(MP_QSTR_choice, MP_OBJ_FROM_PTR(&mod_urandom_choice_obj));
+    #if MICROPY_PY_BUILTINS_FLOAT
+    mp_store_global(MP_QSTR_random, MP_OBJ_FROM_PTR(&mod_urandom_random_obj));
+    mp_store_global(MP_QSTR_uniform, MP_OBJ_FROM_PTR(&mod_urandom_uniform_obj));
+    #endif
+    #endif
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/ure/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in ure so it can coexist)
+MOD = ure_$(ARCH)
+
+# Source files (.c or .py)
+SRC = ure.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/ure/ure.c

@@ -0,0 +1,78 @@
+#define MICROPY_STACK_CHECK (1)
+#define MICROPY_PY_URE (1)
+#define MICROPY_PY_URE_MATCH_GROUPS (1)
+#define MICROPY_PY_URE_MATCH_SPAN_START_END (1)
+#define MICROPY_PY_URE_SUB (0) // requires vstr interface
+
+#include <alloca.h>
+#include "py/dynruntime.h"
+
+#define STACK_LIMIT (2048)
+
+const char *stack_top;
+
+void mp_stack_check(void) {
+    // Assumes descending stack on target
+    volatile char dummy;
+    if (stack_top - &dummy >= STACK_LIMIT) {
+        mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("maximum recursion depth exceeded"));
+    }
+}
+
+#if !defined(__linux__)
+void *memcpy(void *dst, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dst, src, n);
+}
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+void *memmove(void *dest, const void *src, size_t n) {
+    return mp_fun_table.memmove_(dest, src, n);
+}
+
+mp_obj_type_t match_type;
+mp_obj_type_t re_type;
+
+#include "extmod/modure.c"
+
+mp_map_elem_t match_locals_dict_table[5];
+STATIC MP_DEFINE_CONST_DICT(match_locals_dict, match_locals_dict_table);
+
+mp_map_elem_t re_locals_dict_table[3];
+STATIC MP_DEFINE_CONST_DICT(re_locals_dict, re_locals_dict_table);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    char dummy;
+    stack_top = &dummy;
+
+    // Because MP_QSTR_start/end/split are static, xtensa and xtensawin will make a small data section
+    // to copy in this key/value pair if they are specified as a struct, so assign them separately.
+
+    match_type.base.type = (void*)&mp_fun_table.type_type;
+    match_type.name = MP_QSTR_match;
+    match_type.print = match_print;
+    match_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_group), MP_OBJ_FROM_PTR(&match_group_obj) };
+    match_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_groups), MP_OBJ_FROM_PTR(&match_groups_obj) };
+    match_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_span), MP_OBJ_FROM_PTR(&match_span_obj) };
+    match_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_start), MP_OBJ_FROM_PTR(&match_start_obj) };
+    match_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_OBJ_FROM_PTR(&match_end_obj) };
+    match_type.locals_dict = (void*)&match_locals_dict;
+
+    re_type.base.type = (void*)&mp_fun_table.type_type;
+    re_type.name = MP_QSTR_ure;
+    re_type.print = re_print;
+    re_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) };
+    re_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) };
+    re_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_split), MP_OBJ_FROM_PTR(&re_split_obj) };
+    re_type.locals_dict = (void*)&re_locals_dict;
+
+    mp_store_global(MP_QSTR_compile, MP_OBJ_FROM_PTR(&mod_re_compile_obj));
+    mp_store_global(MP_QSTR_match, MP_OBJ_FROM_PTR(&re_match_obj));
+    mp_store_global(MP_QSTR_search, MP_OBJ_FROM_PTR(&re_search_obj));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

examples/natmod/uzlib/Makefile

@@ -0,0 +1,13 @@
+# Location of top-level MicroPython directory
+MPY_DIR = ../../..
+
+# Name of module (different to built-in uzlib so it can coexist)
+MOD = uzlib_$(ARCH)
+
+# Source files (.c or .py)
+SRC = uzlib.c
+
+# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin)
+ARCH = x64
+
+include $(MPY_DIR)/py/dynruntime.mk

examples/natmod/uzlib/uzlib.c

@@ -0,0 +1,40 @@
+#define MICROPY_PY_UZLIB (1)
+
+#include "py/dynruntime.h"
+
+#if !defined(__linux__)
+void *memset(void *s, int c, size_t n) {
+    return mp_fun_table.memset_(s, c, n);
+}
+#endif
+
+mp_obj_full_type_t decompio_type;
+mp_stream_p_t decompio_stream_p;
+
+#include "extmod/moduzlib.c"
+
+mp_map_elem_t decompio_locals_dict_table[3];
+STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table);
+
+mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
+    MP_DYNRUNTIME_INIT_ENTRY
+
+    decompio_stream_p.name = MP_QSTR_protocol_stream;
+    decompio_stream_p.read = decompio_read;
+
+    decompio_type.base.type = mp_fun_table.type_type;
+    decompio_type.flags = MP_TYPE_FLAG_EXTENDED;
+    decompio_type.name = MP_QSTR_DecompIO;
+    decompio_type.make_new = decompio_make_new;
+    decompio_type.ext[0].protocol = &decompio_stream_p;
+    decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) };
+    decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) };
+    decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) };
+    decompio_type.locals_dict = (void*)&decompio_locals_dict;
+
+    mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uzlib));
+    mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_uzlib_decompress_obj));
+    mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type));
+
+    MP_DYNRUNTIME_INIT_EXIT
+}

py/dynruntime.h

@@ -175,6 +175,12 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) {
     return mp_fun_table.call_function_n_kw(mp_fun_table.load_name(MP_QSTR_len), 1, &o);
 }
 
+static inline void *mp_obj_malloc_helper_dyn(size_t num_bytes, const mp_obj_type_t *type) {
+    mp_obj_base_t *base = (mp_obj_base_t *)m_malloc(num_bytes);
+    base->type = type;
+    return base;
+}
+
 /******************************************************************************/
 // General runtime functions
 

py/persistentcode.c

@@ -389,7 +389,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
 mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) {
     byte header[4];
     read_bytes(reader, header, sizeof(header));
-    if (header[0] != 'M'
+    if (header[0] != 'C'
         || header[1] != MPY_VERSION
         || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS
         || header[3] > MP_SMALL_INT_BITS) {
@@ -579,12 +579,12 @@ STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) {
 
 void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
     // header contains:
-    //  byte  'M'
+    //  byte  'C'
     //  byte  version
     //  byte  feature flags
     //  byte  number of bits in a small int
     byte header[4] = {
-        'M',
+        'C',
         MPY_VERSION,
         MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC),
         #if MICROPY_DYNAMIC_COMPILER

py/runtime0.h

@@ -29,6 +29,7 @@
 
 // The first four must fit in 8 bits, see emitbc.c
 // The remaining must fit in 16 bits, see scope.h
+// and must match definitions in mpy-tool.py and mpy_ld.py
 #define MP_SCOPE_FLAG_ALL_SIG      (0x1f)
 #define MP_SCOPE_FLAG_GENERATOR    (0x01)
 #define MP_SCOPE_FLAG_VARKEYWORDS  (0x02)

tests/micropython/import_mpy_invalid.py

@@ -50,8 +50,8 @@ class UserFS:
 # these are the test .mpy files
 user_files = {
     "/mod0.mpy": b"",  # empty file
-    "/mod1.mpy": b"M",  # too short header
-    "/mod2.mpy": b"M\x00\x00\x00",  # bad version
+    "/mod1.mpy": b"C",  # too short header
+    "/mod2.mpy": b"C\x00\x00\x00",  # bad version
 }
 
 # create and mount a user filesystem

tests/micropython/import_mpy_native.py

@@ -50,11 +50,11 @@ class UserFS:
 
 
 # these are the test .mpy files
-valid_header = bytes([77, 6, mpy_arch, 31])
+valid_header = bytes([ord("C"), 6, mpy_arch, 31])
 # fmt: off
 user_files = {
     # bad architecture
-    '/mod0.mpy': b'M\x06\xfc\x1f',
+    '/mod0.mpy': b'C\x06\xfc\x1f',
 
     # test loading of viper and asm
     '/mod1.mpy': valid_header + (
@@ -99,7 +99,7 @@ user_files = {
 
             b'\x22' # 4 bytes, no children, viper code
                 b'\x00\x00\x00\x00' # dummy machine code
-                b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC
+                b'\xe0' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC
                 b'\x06\x04' # rodata=6 bytes, bss=4 bytes
                 b'rodata' # rodata content
                 b'\x03\x01\x00' # dummy relocation of rodata

tests/micropython/import_mpy_native_gc.py

@@ -1,9 +1,5 @@
 # Test that native code loaded from a .mpy file is retained after a GC.
 
-# This is known not to work in CircuitPython. Fixes welcome.
-print("SKIP")
-raise SystemExit
-
 try:
     import gc, sys, uio, uos
 
@@ -53,9 +49,9 @@ class UserFS:
 # by the required value of sys.implementation._mpy.
 features0_file_contents = {
     # -march=x64
-    0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
+    0x806: b'C\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11$\r&\xa5 \x01"\xff',
     # -march=armv6m
-    0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
+    0x1006: b'C\x06\x14\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88"\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfn\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05F\x07K\x08I\xf2XyDP\x88ck\x98G(F\xb8G h\xf8\xbd\x00\xbf:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11>\r@\xa5:\x01<\xff',
 }
 
 # Populate armv7m-derived archs based on armv6m.

tools/analyze_mpy.py

@@ -316,8 +316,8 @@ class mpyFile:
     def __init__(self, encoded_mpy):
         # this matches mp-raw_code_save in py/persistentcode.c
         first_byte = encoded_mpy.read(1)
-        if first_byte != b"M":
-            raise ValueError("Not a valid first byte. Should be 'M' but is {}".format(first_byte))
+        if first_byte != b"C":
+            raise ValueError("Not a valid first byte. Should be 'C' but is {}".format(first_byte))
         self.version = encoded_mpy.read(1)[0]
         self.feature_flags = encoded_mpy.read(1)[0]
         self.small_int_bits = encoded_mpy.read(1)[0]

tools/mpy-tool.py

@@ -125,6 +125,7 @@ MP_PERSISTENT_OBJ_FLOAT = 8
 MP_PERSISTENT_OBJ_COMPLEX = 9
 MP_PERSISTENT_OBJ_TUPLE = 10
 
+# Circuitpython: this does not match upstream because we added MP_SCOPE_FLAG_ASYNC
 MP_SCOPE_FLAG_VIPERRELOC = 0x10
 MP_SCOPE_FLAG_VIPERRODATA = 0x20
 MP_SCOPE_FLAG_VIPERBSS = 0x40
@@ -1328,7 +1329,7 @@ def read_mpy(filename):
 
         # Read and verify the header.
         header = reader.read_bytes(4)
-        if header[0] != ord("M"):
+        if header[0] != ord("C"):
             raise MPYReadError(filename, "not a valid .mpy file")
         if header[1] != config.MPY_VERSION:
             raise MPYReadError(filename, "incompatible .mpy version")
@@ -1668,7 +1669,7 @@ def merge_mpy(compiled_modules, output_file):
             compiled_modules.insert(0, compiled_modules.pop(main_cm_idx))
 
         header = bytearray(4)
-        header[0] = ord("M")
+        header[0] = ord("C")
         header[1] = config.MPY_VERSION
         header[2] = config.native_arch << 2
         header[3] = config.mp_small_int_bits

tools/mpy_ld.py

@@ -47,9 +47,10 @@ MP_NATIVE_ARCH_ARMV7EMDP = 8
 MP_NATIVE_ARCH_XTENSA = 9
 MP_NATIVE_ARCH_XTENSAWIN = 10
 MP_PERSISTENT_OBJ_STR = 5
-MP_SCOPE_FLAG_VIPERRELOC = 0x10
-MP_SCOPE_FLAG_VIPERRODATA = 0x20
-MP_SCOPE_FLAG_VIPERBSS = 0x40
+# Circuitpython: this does not match upstream because we added MP_SCOPE_FLAG_ASYNC
+MP_SCOPE_FLAG_VIPERRELOC = 0x20
+MP_SCOPE_FLAG_VIPERRODATA = 0x40
+MP_SCOPE_FLAG_VIPERBSS = 0x80
 MP_SMALL_INT_BITS = 31
 
 # ELF constants
@@ -768,7 +769,8 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len):
     # Resolve unknown symbols
     mp_fun_table_sec = Section(".external.mp_fun_table", b"", 0)
     fun_table = {
-        key: 67 + idx
+        # Circuitpython: this does not match upstream because we added an item in _mp_fnu_table_t
+        key: 68 + idx
         for idx, key in enumerate(
             [
                 "mp_type_type",
@@ -917,7 +919,7 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
     out.open(fmpy)
 
     # MPY: header
-    out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS]))
+    out.write_bytes(bytearray([ord("C"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS]))
 
     # MPY: n_qstr
     out.write_uint(1 + len(native_qstr_vals))

tools/pyboard.py

@@ -748,7 +748,7 @@ def main():
         for filename in args.files:
             with open(filename, "rb") as f:
                 pyfile = f.read()
-                if filename.endswith(".mpy") and pyfile[0] == ord("M"):
+                if filename.endswith(".mpy") and pyfile[0] == ord("C"):
                     pyb.exec_("_injected_buf=" + repr(pyfile))
                     pyfile = _injected_import_hook_code
                 execbuffer(pyfile)