Merge pull request #6028 from jepler/fix-test-natmod

Fix all natmod tests to run on x64

.github/workflows/build.yml

@@ -77,6 +77,19 @@ jobs:
     - name: Native mpy Tests
       run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-tests.py -j1 --mpy-cross-flags='-mcache-lookup-bc' --via-mpy --emit native -d basics float micropython
       working-directory: tests
+    - name: Build native modules
+      run: |
+        make -C examples/natmod/features1
+        make -C examples/natmod/features2
+        make -C examples/natmod/btree
+        make -C examples/natmod/framebuf
+        make -C examples/natmod/uheapq
+        make -C examples/natmod/urandom
+        make -C examples/natmod/ure
+        make -C examples/natmod/uzlib
+    - name: Test native modules
+      run: MICROPY_CPYTHON3=python3.8 MICROPY_MICROPYTHON=../ports/unix/micropython-coverage ./run-natmodtests.py extmod/{btree*,framebuf*,uheapq*,ure*,uzlib*}.py
+      working-directory: tests
     - name: Build mpy-cross.static-aarch64
       run: make -C mpy-cross -j2 -f Makefile.static-aarch64
     - uses: actions/upload-artifact@v2

examples/natmod/btree/btree_c.c

@@ -18,7 +18,7 @@ void *memmove(void *dest, const void *src, size_t n) {
 }
 
 void *malloc(size_t n) {
-    void *ptr = m_malloc(n);
+    void *ptr = m_malloc(n, false);
     return ptr;
 }
 void *realloc(void *ptr, size_t n) {
@@ -26,7 +26,7 @@ void *realloc(void *ptr, size_t n) {
     return NULL;
 }
 void *calloc(size_t n, size_t m) {
-    void *ptr = m_malloc(n * m);
+    void *ptr = m_malloc(n * m, false);
     // memory already cleared by conservative GC
     return ptr;
 }
@@ -51,7 +51,7 @@ int *__errno (void)
 
 ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) {
     mp_obj_base_t* o = stream;
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = o->type->ext[0].protocol;
     mp_uint_t out_sz = stream_p->write(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
     if (out_sz == MP_STREAM_ERROR) {
         return -1;
@@ -62,7 +62,7 @@ ssize_t mp_stream_posix_write(void *stream, const void *buf, size_t len) {
 
 ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) {
     mp_obj_base_t* o = stream;
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = o->type->ext[0].protocol;
     mp_uint_t out_sz = stream_p->read(MP_OBJ_FROM_PTR(stream), buf, len, &native_errno);
     if (out_sz == MP_STREAM_ERROR) {
         return -1;
@@ -73,7 +73,7 @@ ssize_t mp_stream_posix_read(void *stream, void *buf, size_t len) {
 
 off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) {
     const mp_obj_base_t* o = stream;
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = o->type->ext[0].protocol;
     struct mp_stream_seek_t seek_s;
     seek_s.offset = offset;
     seek_s.whence = whence;
@@ -86,7 +86,7 @@ off_t mp_stream_posix_lseek(void *stream, off_t offset, int whence) {
 
 int mp_stream_posix_fsync(void *stream) {
     mp_obj_base_t* o = stream;
-    const mp_stream_p_t *stream_p = o->type->protocol;
+    const mp_stream_p_t *stream_p = o->type->ext[0].protocol;
     mp_uint_t res = stream_p->ioctl(MP_OBJ_FROM_PTR(stream), MP_STREAM_FLUSH, 0, &native_errno);
     if (res == MP_STREAM_ERROR) {
         return -1;
@@ -94,7 +94,7 @@ int mp_stream_posix_fsync(void *stream) {
     return res;
 }
 
-mp_obj_type_t btree_type;
+mp_obj_full_type_t btree_type;
 
 #include "extmod/modbtree.c"
 
@@ -123,12 +123,13 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a
     MP_DYNRUNTIME_INIT_ENTRY
 
     btree_type.base.type = (void*)&mp_fun_table.type_type;
+    btree_type.flags = MP_TYPE_FLAG_EXTENDED;
     btree_type.name = MP_QSTR_btree;
     btree_type.print = btree_print;
-    btree_type.getiter = btree_getiter;
-    btree_type.iternext = btree_iternext;
-    btree_type.binary_op = btree_binary_op;
-    btree_type.subscr = btree_subscr;
+    btree_type.ext[0].getiter = btree_getiter;
+    btree_type.ext[0].iternext = btree_iternext;
+    btree_type.ext[0].binary_op = btree_binary_op;
+    btree_type.ext[0].subscr = btree_subscr;
     btree_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&btree_close_obj) };
     btree_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_flush), MP_OBJ_FROM_PTR(&btree_flush_obj) };
     btree_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_get), MP_OBJ_FROM_PTR(&btree_get_obj) };

examples/natmod/framebuf/framebuf.c

@@ -8,7 +8,7 @@ void *memset(void *s, int c, size_t n) {
 }
 #endif
 
-mp_obj_type_t mp_type_framebuf;
+mp_obj_full_type_t mp_type_framebuf;
 
 #include "extmod/modframebuf.c"
 
@@ -19,9 +19,10 @@ mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *a
     MP_DYNRUNTIME_INIT_ENTRY
 
     mp_type_framebuf.base.type = (void*)&mp_type_type;
+    mp_type_framebuf.flags = MP_TYPE_FLAG_EXTENDED;
     mp_type_framebuf.name = MP_QSTR_FrameBuffer;
     mp_type_framebuf.make_new = framebuf_make_new;
-    mp_type_framebuf.buffer_p.get_buffer = framebuf_get_buffer;
+    mp_type_framebuf.ext[0].buffer_p.get_buffer = framebuf_get_buffer;
     framebuf_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill), MP_OBJ_FROM_PTR(&framebuf_fill_obj) };
     framebuf_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_fill_rect), MP_OBJ_FROM_PTR(&framebuf_fill_rect_obj) };
     framebuf_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_pixel), MP_OBJ_FROM_PTR(&framebuf_pixel_obj) };

examples/natmod/uzlib/uzlib.c

@@ -8,7 +8,8 @@ void *memset(void *s, int c, size_t n) {
 }
 #endif
 
-mp_obj_type_t decompio_type;
+mp_obj_full_type_t decompio_type;
+mp_stream_p_t decompio_stream_p;
 
 #include "extmod/moduzlib.c"
 
@@ -18,10 +19,14 @@ STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table);
 mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) {
     MP_DYNRUNTIME_INIT_ENTRY
 
+    decompio_stream_p.name = MP_QSTR_protocol_stream;
+    decompio_stream_p.read = decompio_read;
+
     decompio_type.base.type = mp_fun_table.type_type;
+    decompio_type.flags = MP_TYPE_FLAG_EXTENDED;
     decompio_type.name = MP_QSTR_DecompIO;
     decompio_type.make_new = decompio_make_new;
-    decompio_type.protocol = &decompio_stream_p;
+    decompio_type.ext[0].protocol = &decompio_stream_p;
     decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) };
     decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) };
     decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) };

extmod/modbtree.c

@@ -47,7 +47,7 @@ void __dbpanic(DB *db) {
 
 STATIC mp_obj_btree_t *btree_new(DB *db, mp_obj_t stream) {
     mp_obj_btree_t *o = m_new_obj(mp_obj_btree_t);
-    o->base.type = &btree_type;
+    o->base.type = (mp_obj_type_t *)&btree_type;
     o->stream = stream;
     o->db = db;
     o->start_key = mp_const_none;

extmod/modframebuf.c

@@ -622,7 +622,7 @@ STATIC const mp_obj_type_t mp_type_framebuf = {
 // this factory function is provided for backwards compatibility with old FrameBuffer1 class
 STATIC mp_obj_t legacy_framebuffer1(size_t n_args, const mp_obj_t *args) {
     mp_obj_framebuf_t *o = m_new_obj(mp_obj_framebuf_t);
-    o->base.type = &mp_type_framebuf;
+    o->base.type = (mp_obj_type_t *)&mp_type_framebuf;
 
     mp_buffer_info_t bufinfo;
     mp_get_buffer_raise(args[0], &bufinfo, MP_BUFFER_WRITE);

extmod/moduzlib.c

@@ -114,12 +114,12 @@ STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = {
 STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table);
 #endif
 
+#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_stream_p_t decompio_stream_p = {
     MP_PROTO_IMPLEMENT(MP_QSTR_protocol_stream)
     .read = decompio_read,
 };
 
-#if !MICROPY_ENABLE_DYNRUNTIME
 STATIC const mp_obj_type_t decompio_type = {
     { &mp_type_type },
     .flags = MP_TYPE_FLAG_EXTENDED,

py/dynruntime.h

@@ -141,8 +141,7 @@ static inline mp_obj_t mp_obj_cast_to_native_base_dyn(mp_obj_t self_in, mp_const
     if (MP_OBJ_FROM_PTR(self_type) == native_type) {
         return self_in;
     }
-    mp_parent_t parent = mp_type_get_parent_slot(self_type);
-    if (parent != native_type) {
+    if (self_type->parent != native_type) {
         // The self_in object is not a direct descendant of native_type, so fail the cast.
         // This is a very simple version of mp_obj_is_subclass_fast that could be improved.
         return MP_OBJ_NULL;

requirements-dev.txt

@@ -21,3 +21,6 @@ black
 
 # for combining the Nordic SoftDevice with CircuitPython
 intelhex
+
+# for building & testing natmods
+pyelftools

tests/run-natmodtests.py

@@ -9,9 +9,6 @@ import subprocess
 import sys
 import argparse
 
-sys.path.append("../tools")
-import pyboard
-
 # Paths for host executables
 CPYTHON3 = os.getenv("MICROPY_CPYTHON3", "python3")
 MICROPYTHON = os.getenv("MICROPY_MICROPYTHON", "../ports/unix/micropython-coverage")
@@ -178,6 +175,10 @@ def main():
     target_truth = TargetSubprocess([CPYTHON3])
 
     if args.pyboard:
+        global pyboard
+        sys.path.append("../tools")
+        import pyboard
+
         target = TargetPyboard(pyboard.Pyboard(args.device))
     else:
         target = TargetSubprocess([MICROPYTHON])