From a94301122a1cdc3fb85fb0667576f31b709b2ee6 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 19 Aug 2023 16:50:04 -0500 Subject: [PATCH] Make natmods work again. And put back our magic number, because our bytecode format differs from upstream drop btree & framebuf natmods, they had additional problems I didn't want to fix right now. --- .github/workflows/run-tests.yml | 2 - docs/reference/mpyfiles.rst | 4 +- examples/natmod/.gitignore | 1 + examples/natmod/features0/Makefile | 14 +++ examples/natmod/features0/features0.c | 40 ++++++++ examples/natmod/features1/Makefile | 14 +++ examples/natmod/features1/features1.c | 106 ++++++++++++++++++++++ examples/natmod/features2/Makefile | 14 +++ examples/natmod/features2/main.c | 83 +++++++++++++++++ examples/natmod/features2/prod.c | 9 ++ examples/natmod/features2/prod.h | 1 + examples/natmod/features2/test.py | 29 ++++++ examples/natmod/features3/Makefile | 14 +++ examples/natmod/features3/features3.c | 60 ++++++++++++ examples/natmod/uheapq/Makefile | 13 +++ examples/natmod/uheapq/uheapq.c | 16 ++++ examples/natmod/urandom/Makefile | 13 +++ examples/natmod/urandom/urandom.c | 33 +++++++ examples/natmod/ure/Makefile | 13 +++ examples/natmod/ure/ure.c | 78 ++++++++++++++++ examples/natmod/uzlib/Makefile | 13 +++ examples/natmod/uzlib/uzlib.c | 40 ++++++++ py/dynruntime.h | 6 ++ py/persistentcode.c | 6 +- py/runtime0.h | 1 + tests/micropython/import_mpy_invalid.py | 4 +- tests/micropython/import_mpy_native.py | 6 +- tests/micropython/import_mpy_native_gc.py | 8 +- tools/analyze_mpy.py | 4 +- tools/mpy-tool.py | 5 +- tools/mpy_ld.py | 12 ++- tools/pyboard.py | 2 +- 32 files changed, 636 insertions(+), 28 deletions(-) create mode 100644 examples/natmod/.gitignore create mode 100644 examples/natmod/features0/Makefile create mode 100644 examples/natmod/features0/features0.c create mode 100644 examples/natmod/features1/Makefile create mode 100644 examples/natmod/features1/features1.c create mode 100644 examples/natmod/features2/Makefile create mode 100644 examples/natmod/features2/main.c create mode 100644 examples/natmod/features2/prod.c create mode 100644 examples/natmod/features2/prod.h create mode 100644 examples/natmod/features2/test.py create mode 100644 examples/natmod/features3/Makefile create mode 100644 examples/natmod/features3/features3.c create mode 100644 examples/natmod/uheapq/Makefile create mode 100644 examples/natmod/uheapq/uheapq.c create mode 100644 examples/natmod/urandom/Makefile create mode 100644 examples/natmod/urandom/urandom.c create mode 100644 examples/natmod/ure/Makefile create mode 100644 examples/natmod/ure/ure.c create mode 100644 examples/natmod/uzlib/Makefile create mode 100644 examples/natmod/uzlib/uzlib.c diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 44146662f2..76422b1c3d 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -55,8 +55,6 @@ jobs: run: | make -C examples/natmod/features1 make -C examples/natmod/features2 - make -C examples/natmod/btree - make -C examples/natmod/framebuf make -C examples/natmod/uheapq make -C examples/natmod/urandom make -C examples/natmod/ure diff --git a/docs/reference/mpyfiles.rst b/docs/reference/mpyfiles.rst index 4995672b52..b65f62f9ca 100644 --- a/docs/reference/mpyfiles.rst +++ b/docs/reference/mpyfiles.rst @@ -66,7 +66,7 @@ If importing an .mpy file fails then try the following: print() * Check the validity of the .mpy file by inspecting the first two bytes of - the file. The first byte should be an uppercase 'M' and the second byte + the file. The first byte should be an uppercase 'C' and the second byte will be the version number, which should match the system version from above. If it doesn't match then rebuild the .mpy file. @@ -144,7 +144,7 @@ The .mpy header is: ====== ================================ size field ====== ================================ -byte value 0x4d (ASCII 'M') +byte value 0x43 (ASCII 'C') byte .mpy version number byte feature flags byte number of bits in a small int diff --git a/examples/natmod/.gitignore b/examples/natmod/.gitignore new file mode 100644 index 0000000000..4815d20f06 --- /dev/null +++ b/examples/natmod/.gitignore @@ -0,0 +1 @@ +*.mpy diff --git a/examples/natmod/features0/Makefile b/examples/natmod/features0/Makefile new file mode 100644 index 0000000000..57490df90a --- /dev/null +++ b/examples/natmod/features0/Makefile @@ -0,0 +1,14 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module +MOD = features0 + +# Source files (.c or .py) +SRC = features0.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +# Include to get the rules for compiling and linking the module +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features0/features0.c b/examples/natmod/features0/features0.c new file mode 100644 index 0000000000..1b1867a3bc --- /dev/null +++ b/examples/natmod/features0/features0.c @@ -0,0 +1,40 @@ +/* This example demonstrates the following features in a native module: + - defining a simple function exposed to Python + - defining a local, helper C function + - getting and creating integer objects +*/ + +// Include the header file to get access to the MicroPython API +#include "py/dynruntime.h" + +// Helper function to compute factorial +STATIC mp_int_t factorial_helper(mp_int_t x) { + if (x == 0) { + return 1; + } + return x * factorial_helper(x - 1); +} + +// This is the function which will be called from Python, as factorial(x) +STATIC mp_obj_t factorial(mp_obj_t x_obj) { + // Extract the integer from the MicroPython input object + mp_int_t x = mp_obj_get_int(x_obj); + // Calculate the factorial + mp_int_t result = factorial_helper(x); + // Convert the result to a MicroPython integer object and return it + return mp_obj_new_int(result); +} +// Define a Python reference to the function above +STATIC MP_DEFINE_CONST_FUN_OBJ_1(factorial_obj, factorial); + +// This is the entry point and is called when the module is imported +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // This must be first, it sets up the globals dict and other things + MP_DYNRUNTIME_INIT_ENTRY + + // Make the function available in the module's namespace + mp_store_global(MP_QSTR_factorial, MP_OBJ_FROM_PTR(&factorial_obj)); + + // This must be last, it restores the globals dict + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/features1/Makefile b/examples/natmod/features1/Makefile new file mode 100644 index 0000000000..010640daf9 --- /dev/null +++ b/examples/natmod/features1/Makefile @@ -0,0 +1,14 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module +MOD = features1 + +# Source files (.c or .py) +SRC = features1.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +# Include to get the rules for compiling and linking the module +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features1/features1.c b/examples/natmod/features1/features1.c new file mode 100644 index 0000000000..a5e82252a1 --- /dev/null +++ b/examples/natmod/features1/features1.c @@ -0,0 +1,106 @@ +/* This example demonstrates the following features in a native module: + - defining simple functions exposed to Python + - defining local, helper C functions + - defining constant integers and strings exposed to Python + - getting and creating integer objects + - creating Python lists + - raising exceptions + - allocating memory + - BSS and constant data (rodata) + - relocated pointers in rodata +*/ + +// Include the header file to get access to the MicroPython API +#include "py/dynruntime.h" + +// BSS (zero) data +uint16_t data16[4]; + +// Constant data (rodata) +const uint8_t table8[] = { 0, 1, 1, 2, 3, 5, 8, 13 }; +const uint16_t table16[] = { 0x1000, 0x2000 }; + +// Constant data pointing to BSS/constant data +uint16_t *const table_ptr16a[] = { &data16[0], &data16[1], &data16[2], &data16[3] }; +const uint16_t *const table_ptr16b[] = { &table16[0], &table16[1] }; + +// A simple function that adds its 2 arguments (must be integers) +STATIC mp_obj_t add(mp_obj_t x_in, mp_obj_t y_in) { + mp_int_t x = mp_obj_get_int(x_in); + mp_int_t y = mp_obj_get_int(y_in); + return mp_obj_new_int(x + y); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add); + +// A local helper function (not exposed to Python) +STATIC mp_int_t fibonacci_helper(mp_int_t x) { + if (x < MP_ARRAY_SIZE(table8)) { + return table8[x]; + } else { + return fibonacci_helper(x - 1) + fibonacci_helper(x - 2); + } +} + +// A function which computes Fibonacci numbers +STATIC mp_obj_t fibonacci(mp_obj_t x_in) { + mp_int_t x = mp_obj_get_int(x_in); + if (x < 0) { + mp_raise_ValueError(MP_ERROR_TEXT("can't compute negative Fibonacci number")); + } + return mp_obj_new_int(fibonacci_helper(x)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(fibonacci_obj, fibonacci); + +// A function that accesses the BSS data +STATIC mp_obj_t access(size_t n_args, const mp_obj_t *args) { + if (n_args == 0) { + // Create a list holding all items from data16 + mp_obj_list_t *lst = MP_OBJ_TO_PTR(mp_obj_new_list(MP_ARRAY_SIZE(data16), NULL)); + for (int i = 0; i < MP_ARRAY_SIZE(data16); ++i) { + lst->items[i] = mp_obj_new_int(data16[i]); + } + return MP_OBJ_FROM_PTR(lst); + } else if (n_args == 1) { + // Get one item from data16 + mp_int_t idx = mp_obj_get_int(args[0]) & 3; + return mp_obj_new_int(data16[idx]); + } else { + // Set one item in data16 (via table_ptr16a) + mp_int_t idx = mp_obj_get_int(args[0]) & 3; + *table_ptr16a[idx] = mp_obj_get_int(args[1]); + return mp_const_none; + } +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(access_obj, 0, 2, access); + +// A function that allocates memory and creates a bytearray +STATIC mp_obj_t make_array(void) { + uint16_t *ptr = m_new(uint16_t, MP_ARRAY_SIZE(table_ptr16b)); + for (int i = 0; i < MP_ARRAY_SIZE(table_ptr16b); ++i) { + ptr[i] = *table_ptr16b[i]; + } + return mp_obj_new_bytearray_by_ref(sizeof(uint16_t) * MP_ARRAY_SIZE(table_ptr16b), ptr); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(make_array_obj, make_array); + +// This is the entry point and is called when the module is imported +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // This must be first, it sets up the globals dict and other things + MP_DYNRUNTIME_INIT_ENTRY + + // Messages can be printed as usually + mp_printf(&mp_plat_print, "initialising module self=%p\n", self); + + // Make the functions available in the module's namespace + mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj)); + mp_store_global(MP_QSTR_fibonacci, MP_OBJ_FROM_PTR(&fibonacci_obj)); + mp_store_global(MP_QSTR_access, MP_OBJ_FROM_PTR(&access_obj)); + mp_store_global(MP_QSTR_make_array, MP_OBJ_FROM_PTR(&make_array_obj)); + + // Add some constants to the module's namespace + mp_store_global(MP_QSTR_VAL, MP_OBJ_NEW_SMALL_INT(42)); + mp_store_global(MP_QSTR_MSG, MP_OBJ_NEW_QSTR(MP_QSTR_HELLO_MICROPYTHON)); + + // This must be last, it restores the globals dict + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/features2/Makefile b/examples/natmod/features2/Makefile new file mode 100644 index 0000000000..4fd23c6879 --- /dev/null +++ b/examples/natmod/features2/Makefile @@ -0,0 +1,14 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module +MOD = features2 + +# Source files (.c or .py) +SRC = main.c prod.c test.py + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +# Include to get the rules for compiling and linking the module +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features2/main.c b/examples/natmod/features2/main.c new file mode 100644 index 0000000000..1a39700dc4 --- /dev/null +++ b/examples/natmod/features2/main.c @@ -0,0 +1,83 @@ +/* This example demonstrates the following features in a native module: + - using floats + - defining additional code in Python (see test.py) + - have extra C code in a separate file (see prod.c) +*/ + +// Include the header file to get access to the MicroPython API +#include "py/dynruntime.h" + +// Include the header for auxiliary C code for this module +#include "prod.h" + +// Automatically detect if this module should include double-precision code. +// If double precision is supported by the target architecture then it can +// be used in native module regardless of what float setting the target +// MicroPython runtime uses (being none, float or double). +#if defined(__i386__) || defined(__x86_64__) || (defined(__ARM_FP) && (__ARM_FP & 8)) +#define USE_DOUBLE 1 +#else +#define USE_DOUBLE 0 +#endif + +// A function that uses the default float type configured for the current target +// This default can be overridden by specifying MICROPY_FLOAT_IMPL at the make level +STATIC mp_obj_t add(mp_obj_t x, mp_obj_t y) { + return mp_obj_new_float(mp_obj_get_float(x) + mp_obj_get_float(y)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_obj, add); + +// A function that explicitly uses single precision floats +STATIC mp_obj_t add_f(mp_obj_t x, mp_obj_t y) { + return mp_obj_new_float_from_f(mp_obj_get_float_to_f(x) + mp_obj_get_float_to_f(y)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_f_obj, add_f); + +#if USE_DOUBLE +// A function that explicitly uses double precision floats +STATIC mp_obj_t add_d(mp_obj_t x, mp_obj_t y) { + return mp_obj_new_float_from_d(mp_obj_get_float_to_d(x) + mp_obj_get_float_to_d(y)); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(add_d_obj, add_d); +#endif + +// A function that computes the product of floats in an array. +// This function uses the most general C argument interface, which is more difficult +// to use but has access to the globals dict of the module via self->globals. +STATIC mp_obj_t productf(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // Check number of arguments is valid + mp_arg_check_num(n_args, n_kw, 1, 1, false); + + // Extract buffer pointer and verify typecode + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_RW); + if (bufinfo.typecode != 'f') { + mp_raise_ValueError(MP_ERROR_TEXT("expecting float array")); + } + + // Compute product, store result back in first element of array + float *ptr = bufinfo.buf; + float prod = prod_array(bufinfo.len / sizeof(*ptr), ptr); + ptr[0] = prod; + + return mp_const_none; +} + +// This is the entry point and is called when the module is imported +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // This must be first, it sets up the globals dict and other things + MP_DYNRUNTIME_INIT_ENTRY + + // Make the functions available in the module's namespace + mp_store_global(MP_QSTR_add, MP_OBJ_FROM_PTR(&add_obj)); + mp_store_global(MP_QSTR_add_f, MP_OBJ_FROM_PTR(&add_f_obj)); + #if USE_DOUBLE + mp_store_global(MP_QSTR_add_d, MP_OBJ_FROM_PTR(&add_d_obj)); + #endif + + // The productf function uses the most general C argument interface + mp_store_global(MP_QSTR_productf, MP_DYNRUNTIME_MAKE_FUNCTION(productf)); + + // This must be last, it restores the globals dict + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/features2/prod.c b/examples/natmod/features2/prod.c new file mode 100644 index 0000000000..7791dcad1d --- /dev/null +++ b/examples/natmod/features2/prod.c @@ -0,0 +1,9 @@ +#include "prod.h" + +float prod_array(int n, float *ar) { + float ans = 1; + for (int i = 0; i < n; ++i) { + ans *= ar[i]; + } + return ans; +} diff --git a/examples/natmod/features2/prod.h b/examples/natmod/features2/prod.h new file mode 100644 index 0000000000..f27dd8d033 --- /dev/null +++ b/examples/natmod/features2/prod.h @@ -0,0 +1 @@ +float prod_array(int n, float *ar); diff --git a/examples/natmod/features2/test.py b/examples/natmod/features2/test.py new file mode 100644 index 0000000000..5ac80120d7 --- /dev/null +++ b/examples/natmod/features2/test.py @@ -0,0 +1,29 @@ +# This Python code will be merged with the C code in main.c + +import array + + +def isclose(a, b): + return abs(a - b) < 1e-3 + + +def test(): + tests = [ + isclose(add(0.1, 0.2), 0.3), + isclose(add_f(0.1, 0.2), 0.3), + ] + + ar = array.array("f", [1, 2, 3.5]) + productf(ar) + tests.append(isclose(ar[0], 7)) + + if "add_d" in globals(): + tests.append(isclose(add_d(0.1, 0.2), 0.3)) + + print(tests) + + if not all(tests): + raise SystemExit(1) + + +test() diff --git a/examples/natmod/features3/Makefile b/examples/natmod/features3/Makefile new file mode 100644 index 0000000000..4a5f71b8f2 --- /dev/null +++ b/examples/natmod/features3/Makefile @@ -0,0 +1,14 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module +MOD = features3 + +# Source files (.c or .py) +SRC = features3.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +# Include to get the rules for compiling and linking the module +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/features3/features3.c b/examples/natmod/features3/features3.c new file mode 100644 index 0000000000..20efd67cdc --- /dev/null +++ b/examples/natmod/features3/features3.c @@ -0,0 +1,60 @@ +/* This example demonstrates the following features in a native module: + - using types + - using constant objects + - creating dictionaries +*/ + +// Include the header file to get access to the MicroPython API. +#include "py/dynruntime.h" + +// A function that returns a tuple of object types. +STATIC mp_obj_t get_types(void) { + return mp_obj_new_tuple(9, ((mp_obj_t []) { + MP_OBJ_FROM_PTR(&mp_type_type), + MP_OBJ_FROM_PTR(&mp_type_NoneType), + MP_OBJ_FROM_PTR(&mp_type_bool), + MP_OBJ_FROM_PTR(&mp_type_int), + MP_OBJ_FROM_PTR(&mp_type_str), + MP_OBJ_FROM_PTR(&mp_type_bytes), + MP_OBJ_FROM_PTR(&mp_type_tuple), + MP_OBJ_FROM_PTR(&mp_type_list), + MP_OBJ_FROM_PTR(&mp_type_dict), + })); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(get_types_obj, get_types); + +// A function that returns a tuple of constant objects. +STATIC mp_obj_t get_const_objects(void) { + return mp_obj_new_tuple(5, ((mp_obj_t []) { + mp_const_none, + mp_const_false, + mp_const_true, + mp_const_empty_bytes, + mp_const_empty_tuple, + })); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(get_const_objects_obj, get_const_objects); + +// A function that creates a dictionary from the given arguments. +STATIC mp_obj_t make_dict(size_t n_args, const mp_obj_t *args) { + mp_obj_t dict = mp_obj_new_dict(n_args / 2); + for (; n_args >= 2; n_args -= 2, args += 2) { + mp_obj_dict_store(dict, args[0], args[1]); + } + return dict; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(make_dict_obj, 0, MP_OBJ_FUN_ARGS_MAX, make_dict); + +// This is the entry point and is called when the module is imported. +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + // This must be first, it sets up the globals dict and other things. + MP_DYNRUNTIME_INIT_ENTRY + + // Make the functions available in the module's namespace. + mp_store_global(MP_QSTR_make_dict, MP_OBJ_FROM_PTR(&make_dict_obj)); + mp_store_global(MP_QSTR_get_types, MP_OBJ_FROM_PTR(&get_types_obj)); + mp_store_global(MP_QSTR_get_const_objects, MP_OBJ_FROM_PTR(&get_const_objects_obj)); + + // This must be last, it restores the globals dict. + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/uheapq/Makefile b/examples/natmod/uheapq/Makefile new file mode 100644 index 0000000000..55de3cc081 --- /dev/null +++ b/examples/natmod/uheapq/Makefile @@ -0,0 +1,13 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module (different to built-in uheapq so it can coexist) +MOD = uheapq_$(ARCH) + +# Source files (.c or .py) +SRC = uheapq.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/uheapq/uheapq.c b/examples/natmod/uheapq/uheapq.c new file mode 100644 index 0000000000..9da6bb4ada --- /dev/null +++ b/examples/natmod/uheapq/uheapq.c @@ -0,0 +1,16 @@ +#define MICROPY_PY_UHEAPQ (1) + +#include "py/dynruntime.h" + +#include "extmod/moduheapq.c" + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uheapq)); + mp_store_global(MP_QSTR_heappush, MP_OBJ_FROM_PTR(&mod_uheapq_heappush_obj)); + mp_store_global(MP_QSTR_heappop, MP_OBJ_FROM_PTR(&mod_uheapq_heappop_obj)); + mp_store_global(MP_QSTR_heapify, MP_OBJ_FROM_PTR(&mod_uheapq_heapify_obj)); + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/urandom/Makefile b/examples/natmod/urandom/Makefile new file mode 100644 index 0000000000..3f018baaf7 --- /dev/null +++ b/examples/natmod/urandom/Makefile @@ -0,0 +1,13 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module (different to built-in urandom so it can coexist) +MOD = urandom_$(ARCH) + +# Source files (.c or .py) +SRC = urandom.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/urandom/urandom.c b/examples/natmod/urandom/urandom.c new file mode 100644 index 0000000000..e1fed6a554 --- /dev/null +++ b/examples/natmod/urandom/urandom.c @@ -0,0 +1,33 @@ +#define MICROPY_PY_URANDOM (1) +#define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) + +#include "py/dynruntime.h" + +// Dynamic native modules don't support a data section so these must go in the BSS +uint32_t yasmarang_pad, yasmarang_n, yasmarang_d; +uint8_t yasmarang_dat; + +#include "extmod/modurandom.c" + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + yasmarang_pad = 0xeda4baba; + yasmarang_n = 69; + yasmarang_d = 233; + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_urandom)); + mp_store_global(MP_QSTR_getrandbits, MP_OBJ_FROM_PTR(&mod_urandom_getrandbits_obj)); + mp_store_global(MP_QSTR_seed, MP_OBJ_FROM_PTR(&mod_urandom_seed_obj)); + #if MICROPY_PY_URANDOM_EXTRA_FUNCS + mp_store_global(MP_QSTR_randrange, MP_OBJ_FROM_PTR(&mod_urandom_randrange_obj)); + mp_store_global(MP_QSTR_randint, MP_OBJ_FROM_PTR(&mod_urandom_randint_obj)); + mp_store_global(MP_QSTR_choice, MP_OBJ_FROM_PTR(&mod_urandom_choice_obj)); + #if MICROPY_PY_BUILTINS_FLOAT + mp_store_global(MP_QSTR_random, MP_OBJ_FROM_PTR(&mod_urandom_random_obj)); + mp_store_global(MP_QSTR_uniform, MP_OBJ_FROM_PTR(&mod_urandom_uniform_obj)); + #endif + #endif + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/ure/Makefile b/examples/natmod/ure/Makefile new file mode 100644 index 0000000000..f5254298fd --- /dev/null +++ b/examples/natmod/ure/Makefile @@ -0,0 +1,13 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module (different to built-in ure so it can coexist) +MOD = ure_$(ARCH) + +# Source files (.c or .py) +SRC = ure.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/ure/ure.c b/examples/natmod/ure/ure.c new file mode 100644 index 0000000000..175b93e395 --- /dev/null +++ b/examples/natmod/ure/ure.c @@ -0,0 +1,78 @@ +#define MICROPY_STACK_CHECK (1) +#define MICROPY_PY_URE (1) +#define MICROPY_PY_URE_MATCH_GROUPS (1) +#define MICROPY_PY_URE_MATCH_SPAN_START_END (1) +#define MICROPY_PY_URE_SUB (0) // requires vstr interface + +#include +#include "py/dynruntime.h" + +#define STACK_LIMIT (2048) + +const char *stack_top; + +void mp_stack_check(void) { + // Assumes descending stack on target + volatile char dummy; + if (stack_top - &dummy >= STACK_LIMIT) { + mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("maximum recursion depth exceeded")); + } +} + +#if !defined(__linux__) +void *memcpy(void *dst, const void *src, size_t n) { + return mp_fun_table.memmove_(dst, src, n); +} +void *memset(void *s, int c, size_t n) { + return mp_fun_table.memset_(s, c, n); +} +#endif + +void *memmove(void *dest, const void *src, size_t n) { + return mp_fun_table.memmove_(dest, src, n); +} + +mp_obj_type_t match_type; +mp_obj_type_t re_type; + +#include "extmod/modure.c" + +mp_map_elem_t match_locals_dict_table[5]; +STATIC MP_DEFINE_CONST_DICT(match_locals_dict, match_locals_dict_table); + +mp_map_elem_t re_locals_dict_table[3]; +STATIC MP_DEFINE_CONST_DICT(re_locals_dict, re_locals_dict_table); + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + char dummy; + stack_top = &dummy; + + // Because MP_QSTR_start/end/split are static, xtensa and xtensawin will make a small data section + // to copy in this key/value pair if they are specified as a struct, so assign them separately. + + match_type.base.type = (void*)&mp_fun_table.type_type; + match_type.name = MP_QSTR_match; + match_type.print = match_print; + match_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_group), MP_OBJ_FROM_PTR(&match_group_obj) }; + match_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_groups), MP_OBJ_FROM_PTR(&match_groups_obj) }; + match_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_span), MP_OBJ_FROM_PTR(&match_span_obj) }; + match_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_start), MP_OBJ_FROM_PTR(&match_start_obj) }; + match_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_OBJ_FROM_PTR(&match_end_obj) }; + match_type.locals_dict = (void*)&match_locals_dict; + + re_type.base.type = (void*)&mp_fun_table.type_type; + re_type.name = MP_QSTR_ure; + re_type.print = re_print; + re_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_match), MP_OBJ_FROM_PTR(&re_match_obj) }; + re_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_search), MP_OBJ_FROM_PTR(&re_search_obj) }; + re_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_split), MP_OBJ_FROM_PTR(&re_split_obj) }; + re_type.locals_dict = (void*)&re_locals_dict; + + mp_store_global(MP_QSTR_compile, MP_OBJ_FROM_PTR(&mod_re_compile_obj)); + mp_store_global(MP_QSTR_match, MP_OBJ_FROM_PTR(&re_match_obj)); + mp_store_global(MP_QSTR_search, MP_OBJ_FROM_PTR(&re_search_obj)); + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/examples/natmod/uzlib/Makefile b/examples/natmod/uzlib/Makefile new file mode 100644 index 0000000000..8761caf2dd --- /dev/null +++ b/examples/natmod/uzlib/Makefile @@ -0,0 +1,13 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module (different to built-in uzlib so it can coexist) +MOD = uzlib_$(ARCH) + +# Source files (.c or .py) +SRC = uzlib.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/uzlib/uzlib.c b/examples/natmod/uzlib/uzlib.c new file mode 100644 index 0000000000..6df9944d6a --- /dev/null +++ b/examples/natmod/uzlib/uzlib.c @@ -0,0 +1,40 @@ +#define MICROPY_PY_UZLIB (1) + +#include "py/dynruntime.h" + +#if !defined(__linux__) +void *memset(void *s, int c, size_t n) { + return mp_fun_table.memset_(s, c, n); +} +#endif + +mp_obj_full_type_t decompio_type; +mp_stream_p_t decompio_stream_p; + +#include "extmod/moduzlib.c" + +mp_map_elem_t decompio_locals_dict_table[3]; +STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + decompio_stream_p.name = MP_QSTR_protocol_stream; + decompio_stream_p.read = decompio_read; + + decompio_type.base.type = mp_fun_table.type_type; + decompio_type.flags = MP_TYPE_FLAG_EXTENDED; + decompio_type.name = MP_QSTR_DecompIO; + decompio_type.make_new = decompio_make_new; + decompio_type.ext[0].protocol = &decompio_stream_p; + decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; + decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; + decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; + decompio_type.locals_dict = (void*)&decompio_locals_dict; + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_uzlib)); + mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_uzlib_decompress_obj)); + mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type)); + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/py/dynruntime.h b/py/dynruntime.h index da0317be6d..b7a9a89156 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -175,6 +175,12 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { return mp_fun_table.call_function_n_kw(mp_fun_table.load_name(MP_QSTR_len), 1, &o); } +static inline void *mp_obj_malloc_helper_dyn(size_t num_bytes, const mp_obj_type_t *type) { + mp_obj_base_t *base = (mp_obj_base_t *)m_malloc(num_bytes); + base->type = type; + return base; +} + /******************************************************************************/ // General runtime functions diff --git a/py/persistentcode.c b/py/persistentcode.c index e15dcffdea..74b26cb285 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -389,7 +389,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) { byte header[4]; read_bytes(reader, header, sizeof(header)); - if (header[0] != 'M' + if (header[0] != 'C' || header[1] != MPY_VERSION || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS || header[3] > MP_SMALL_INT_BITS) { @@ -579,12 +579,12 @@ STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { // header contains: - // byte 'M' + // byte 'C' // byte version // byte feature flags // byte number of bits in a small int byte header[4] = { - 'M', + 'C', MPY_VERSION, MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC), #if MICROPY_DYNAMIC_COMPILER diff --git a/py/runtime0.h b/py/runtime0.h index ab048701c4..218820fd70 100644 --- a/py/runtime0.h +++ b/py/runtime0.h @@ -29,6 +29,7 @@ // The first four must fit in 8 bits, see emitbc.c // The remaining must fit in 16 bits, see scope.h +// and must match definitions in mpy-tool.py and mpy_ld.py #define MP_SCOPE_FLAG_ALL_SIG (0x1f) #define MP_SCOPE_FLAG_GENERATOR (0x01) #define MP_SCOPE_FLAG_VARKEYWORDS (0x02) diff --git a/tests/micropython/import_mpy_invalid.py b/tests/micropython/import_mpy_invalid.py index 7bf9417329..894fc582ce 100644 --- a/tests/micropython/import_mpy_invalid.py +++ b/tests/micropython/import_mpy_invalid.py @@ -50,8 +50,8 @@ class UserFS: # these are the test .mpy files user_files = { "/mod0.mpy": b"", # empty file - "/mod1.mpy": b"M", # too short header - "/mod2.mpy": b"M\x00\x00\x00", # bad version + "/mod1.mpy": b"C", # too short header + "/mod2.mpy": b"C\x00\x00\x00", # bad version } # create and mount a user filesystem diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py index 69e1c40798..0dfbca9086 100644 --- a/tests/micropython/import_mpy_native.py +++ b/tests/micropython/import_mpy_native.py @@ -50,11 +50,11 @@ class UserFS: # these are the test .mpy files -valid_header = bytes([77, 6, mpy_arch, 31]) +valid_header = bytes([ord("C"), 6, mpy_arch, 31]) # fmt: off user_files = { # bad architecture - '/mod0.mpy': b'M\x06\xfc\x1f', + '/mod0.mpy': b'C\x06\xfc\x1f', # test loading of viper and asm '/mod1.mpy': valid_header + ( @@ -99,7 +99,7 @@ user_files = { b'\x22' # 4 bytes, no children, viper code b'\x00\x00\x00\x00' # dummy machine code - b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC + b'\xe0' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC b'\x06\x04' # rodata=6 bytes, bss=4 bytes b'rodata' # rodata content b'\x03\x01\x00' # dummy relocation of rodata diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index 4c496585bb..1234184aed 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -1,9 +1,5 @@ # Test that native code loaded from a .mpy file is retained after a GC. -# This is known not to work in CircuitPython. Fixes welcome. -print("SKIP") -raise SystemExit - try: import gc, sys, uio, uos @@ -53,9 +49,9 @@ class UserFS: # by the required value of sys.implementation._mpy. features0_file_contents = { # -march=x64 - 0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', + 0x806: b'C\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11$\r&\xa5 \x01"\xff', # -march=armv6m - 0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", + 0x1006: b'C\x06\x14\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88"\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfn\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05F\x07K\x08I\xf2XyDP\x88ck\x98G(F\xb8G h\xf8\xbd\x00\xbf:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 \x11>\r@\xa5:\x01<\xff', } # Populate armv7m-derived archs based on armv6m. diff --git a/tools/analyze_mpy.py b/tools/analyze_mpy.py index 8c2056e030..3f212e2679 100755 --- a/tools/analyze_mpy.py +++ b/tools/analyze_mpy.py @@ -316,8 +316,8 @@ class mpyFile: def __init__(self, encoded_mpy): # this matches mp-raw_code_save in py/persistentcode.c first_byte = encoded_mpy.read(1) - if first_byte != b"M": - raise ValueError("Not a valid first byte. Should be 'M' but is {}".format(first_byte)) + if first_byte != b"C": + raise ValueError("Not a valid first byte. Should be 'C' but is {}".format(first_byte)) self.version = encoded_mpy.read(1)[0] self.feature_flags = encoded_mpy.read(1)[0] self.small_int_bits = encoded_mpy.read(1)[0] diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 31212fd5bd..c5f56386c9 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -125,6 +125,7 @@ MP_PERSISTENT_OBJ_FLOAT = 8 MP_PERSISTENT_OBJ_COMPLEX = 9 MP_PERSISTENT_OBJ_TUPLE = 10 +# Circuitpython: this does not match upstream because we added MP_SCOPE_FLAG_ASYNC MP_SCOPE_FLAG_VIPERRELOC = 0x10 MP_SCOPE_FLAG_VIPERRODATA = 0x20 MP_SCOPE_FLAG_VIPERBSS = 0x40 @@ -1328,7 +1329,7 @@ def read_mpy(filename): # Read and verify the header. header = reader.read_bytes(4) - if header[0] != ord("M"): + if header[0] != ord("C"): raise MPYReadError(filename, "not a valid .mpy file") if header[1] != config.MPY_VERSION: raise MPYReadError(filename, "incompatible .mpy version") @@ -1668,7 +1669,7 @@ def merge_mpy(compiled_modules, output_file): compiled_modules.insert(0, compiled_modules.pop(main_cm_idx)) header = bytearray(4) - header[0] = ord("M") + header[0] = ord("C") header[1] = config.MPY_VERSION header[2] = config.native_arch << 2 header[3] = config.mp_small_int_bits diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 09ea90dcd1..4d01f52df8 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -47,9 +47,10 @@ MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 MP_PERSISTENT_OBJ_STR = 5 -MP_SCOPE_FLAG_VIPERRELOC = 0x10 -MP_SCOPE_FLAG_VIPERRODATA = 0x20 -MP_SCOPE_FLAG_VIPERBSS = 0x40 +# Circuitpython: this does not match upstream because we added MP_SCOPE_FLAG_ASYNC +MP_SCOPE_FLAG_VIPERRELOC = 0x20 +MP_SCOPE_FLAG_VIPERRODATA = 0x40 +MP_SCOPE_FLAG_VIPERBSS = 0x80 MP_SMALL_INT_BITS = 31 # ELF constants @@ -768,7 +769,8 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): # Resolve unknown symbols mp_fun_table_sec = Section(".external.mp_fun_table", b"", 0) fun_table = { - key: 67 + idx + # Circuitpython: this does not match upstream because we added an item in _mp_fnu_table_t + key: 68 + idx for idx, key in enumerate( [ "mp_type_type", @@ -917,7 +919,7 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.open(fmpy) # MPY: header - out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) + out.write_bytes(bytearray([ord("C"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) # MPY: n_qstr out.write_uint(1 + len(native_qstr_vals)) diff --git a/tools/pyboard.py b/tools/pyboard.py index d2813ad5ad..a521a04a2e 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -748,7 +748,7 @@ def main(): for filename in args.files: with open(filename, "rb") as f: pyfile = f.read() - if filename.endswith(".mpy") and pyfile[0] == ord("M"): + if filename.endswith(".mpy") and pyfile[0] == ord("C"): pyb.exec_("_injected_buf=" + repr(pyfile)) pyfile = _injected_import_hook_code execbuffer(pyfile)