From b37b57821476d9ea80cdcc89d325dcabded3ffb7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 26 May 2022 10:51:29 +1000 Subject: [PATCH] py/persistentcode: Remove remaining native qstr linking support. Support for architecture-specific qstr linking was removed in d4d53e9e114d779523e382c4ea38f0398e880aae, where native code was changed to access qstr values via qstr_table. The only remaining use for the special qstr link table in persistentcode.c is to support native module written in C, linked via mpy_ld.py. But native modules can also use the standard module-level qstr_table (and obj_table) which was introduced in the .mpy file reworking in f2040bfc7ee033e48acef9f289790f3b4e6b74e5. This commit removes the remaining native qstr liking support in persistentcode.c's load_raw_code function, and adds two new relocation options for constants.qstr_table and constants.obj_table. mpy_ld.py is updated to use these relocations options instead of the native qstr link table. Signed-off-by: Damien George --- py/persistentcode.c | 40 +++------- tests/micropython/import_mpy_native.py | 5 +- tests/micropython/import_mpy_native_gc.py | 4 +- tools/mpy-tool.py | 12 --- tools/mpy_ld.py | 91 ++++++++++------------- 5 files changed, 56 insertions(+), 96 deletions(-) diff --git a/py/persistentcode.c b/py/persistentcode.c index d19a817aa5..6304d1ff08 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -71,6 +71,7 @@ STATIC size_t read_uint(mp_reader_t *reader); typedef struct _reloc_info_t { mp_reader_t *reader; + mp_module_context_t *context; uint8_t *rodata; uint8_t *bss; } reloc_info_t; @@ -112,11 +113,17 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { dest = (uintptr_t)ri->bss; } } else if (op == 6) { + // Destination is qstr_table + dest = (uintptr_t)ri->context->constants.qstr_table; + } else if (op == 7) { + // Destination is obj_table + dest = (uintptr_t)ri->context->constants.obj_table; + } else if (op == 8) { // Destination is mp_fun_table itself dest = (uintptr_t)&mp_fun_table; } else { // Destination is an entry in mp_fun_table - dest = ((uintptr_t *)&mp_fun_table)[op - 7]; + dest = ((uintptr_t *)&mp_fun_table)[op - 9]; } while (n--) { *addr_to_adjust++ += dest; @@ -205,7 +212,7 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) { } } -STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { +STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *context) { // Load function kind and data length size_t kind_len = read_uint(reader); int kind = (kind_len & 3) + MP_CODE_BYTECODE; @@ -239,24 +246,6 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { MP_PLAT_ALLOC_EXEC(fun_data_len, (void **)&fun_data, &fun_alloc); read_bytes(reader, fun_data, fun_data_len); - if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) { - // Parse qstr link table and link native code - size_t n_qstr_link = read_uint(reader); - for (size_t i = 0; i < n_qstr_link; ++i) { - size_t off = read_uint(reader); - qstr qst = load_qstr(reader); - uint8_t *dest = fun_data + (off >> 2); - if ((off & 3) == 0) { - // Generic 16-bit link - dest[0] = qst & 0xff; - dest[1] = (qst >> 8) & 0xff; - } else if ((off & 3) == 3) { - // Generic, aligned qstr-object link - *(mp_obj_t *)dest = MP_OBJ_NEW_QSTR(qst); - } - } - } - if (kind == MP_CODE_NATIVE_PY) { // Read prelude offset within fun_data, and extract scope flags. prelude_offset = read_uint(reader); @@ -315,7 +304,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { n_children = read_uint(reader); children = m_new(mp_raw_code_t *, n_children + (kind == MP_CODE_NATIVE_PY)); for (size_t i = 0; i < n_children; ++i) { - children[i] = load_raw_code(reader); + children[i] = load_raw_code(reader, context); } } @@ -349,7 +338,7 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { #endif // Relocate and commit code to executable address space - reloc_info_t ri = {reader, rodata, bss}; + reloc_info_t ri = {reader, context, rodata, bss}; #if defined(MP_PLAT_COMMIT_EXEC) void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri); @@ -429,7 +418,7 @@ mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t * // Load top-level module. mp_compiled_module_t cm2; - cm2.rc = load_raw_code(reader); + cm2.rc = load_raw_code(reader, context); cm2.context = context; #if MICROPY_PERSISTENT_CODE_SAVE @@ -567,11 +556,6 @@ STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { mp_print_bytes(print, rc->fun_data, rc->fun_data_len); #if MICROPY_EMIT_MACHINE_CODE - if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) { - // Save qstr link table for native code - mp_print_uint(print, 0); - } - if (rc->kind == MP_CODE_NATIVE_PY) { // Save prelude size mp_print_uint(print, rc->prelude_offset); diff --git a/tests/micropython/import_mpy_native.py b/tests/micropython/import_mpy_native.py index d7da16581b..449371ddac 100644 --- a/tests/micropython/import_mpy_native.py +++ b/tests/micropython/import_mpy_native.py @@ -74,9 +74,7 @@ user_files = { b'\x02' # 2 children b'\x42' # 8 bytes, no children, viper code - b'\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00' # slot for qstr0 - b'\x01\x0c\x0aprint\x00' # n_qstr=1, qstr0 + b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code b'\x00' # scope_flags b'\x43' # 8 bytes, no children, asm code @@ -102,7 +100,6 @@ user_files = { b'\x22' # 4 bytes, no children, viper code b'\x00\x00\x00\x00' # dummy machine code - b'\x00' # n_qstr=0 b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC b'\x06\x04' # rodata=6 bytes, bss=4 bytes b'rodata' # rodata content diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index aac3375af4..bc8dcafdc7 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -49,9 +49,9 @@ class UserFS: # by the required value of sys.implementation._mpy. features0_file_contents = { # -march=x64 - 0x806: b'M\x06\b\x1f\x01\x004build/features0.native.mpy\x00\x8aB\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x85\x00\x12factorial\x00\x10\r$\x01&\x9f \x01"\xff', + 0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff', # -march=armv6m - 0x1006: b'M\x06\x10\x1f\x01\x004build/features0.native.mpy\x00\x88"\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyD\x18\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial\x00\x10\r<\x01>\x9f8\x01:\xff', + 0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff", } # Populate armv7m-derived archs based on armv6m. diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 6a22f5a404..20ccc87d43 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -976,7 +976,6 @@ class RawCodeNative(RawCode): kind, fun_data, prelude_offset, - qstr_links, scope_flags, n_pos_args, type_sig, @@ -989,7 +988,6 @@ class RawCodeNative(RawCode): self.scope_flags = scope_flags self.n_pos_args = n_pos_args - self.qstr_links = qstr_links self.type_sig = type_sig if config.native_arch in ( MP_NATIVE_ARCH_X86, @@ -1196,15 +1194,6 @@ def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments): rc = RawCodeBytecode(cm_escaped_name, qstr_table, obj_table, fun_data) else: # Create native raw code. - qstr_links = [] - if kind in (MP_CODE_NATIVE_PY, MP_CODE_NATIVE_VIPER): - # Read qstr link table. - n_qstr_link = reader.read_uint() - for _ in range(n_qstr_link): - off = reader.read_uint() - qst = read_qstr(reader, segments) - qstr_links.append((off >> 2, off & 3, qst)) - native_scope_flags = 0 native_n_pos_args = 0 native_type_sig = 0 @@ -1242,7 +1231,6 @@ def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments): kind, fun_data, prelude_offset, - qstr_links, native_scope_flags, native_n_pos_args, native_type_sig, diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index b8652462d4..9fda49826f 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -36,6 +36,8 @@ import makeqstrdata as qstrutil # MicroPython constants MPY_VERSION = 6 +MP_CODE_BYTECODE = 2 +MP_CODE_NATIVE_VIPER = 4 MP_NATIVE_ARCH_X86 = 1 MP_NATIVE_ARCH_X64 = 2 MP_NATIVE_ARCH_ARMV6M = 4 @@ -44,8 +46,7 @@ MP_NATIVE_ARCH_ARMV7EMSP = 7 MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 -MP_CODE_BYTECODE = 2 -MP_CODE_NATIVE_VIPER = 4 +MP_PERSISTENT_OBJ_STR = 5 MP_SCOPE_FLAG_VIPERRELOC = 0x10 MP_SCOPE_FLAG_VIPERRODATA = 0x20 MP_SCOPE_FLAG_VIPERBSS = 0x40 @@ -110,21 +111,20 @@ def asm_jump_xtensa(entry): class ArchData: - def __init__(self, name, mpy_feature, qstr_entry_size, word_size, arch_got, asm_jump): + def __init__(self, name, mpy_feature, word_size, arch_got, asm_jump, *, separate_rodata=False): self.name = name self.mpy_feature = mpy_feature - self.qstr_entry_size = qstr_entry_size + self.qstr_entry_size = 2 self.word_size = word_size self.arch_got = arch_got self.asm_jump = asm_jump - self.separate_rodata = name == "EM_XTENSA" and qstr_entry_size == 4 + self.separate_rodata = separate_rodata ARCH_DATA = { "x86": ArchData( "EM_386", MP_NATIVE_ARCH_X86 << 2, - 2, 4, (R_386_PC32, R_386_GOT32, R_386_GOT32X), asm_jump_x86, @@ -132,7 +132,6 @@ ARCH_DATA = { "x64": ArchData( "EM_X86_64", MP_NATIVE_ARCH_X64 << 2, - 2, 8, (R_X86_64_GOTPCREL, R_X86_64_REX_GOTPCRELX), asm_jump_x86, @@ -140,7 +139,6 @@ ARCH_DATA = { "armv6m": ArchData( "EM_ARM", MP_NATIVE_ARCH_ARMV6M << 2, - 2, 4, (R_ARM_GOT_BREL,), asm_jump_thumb, @@ -148,7 +146,6 @@ ARCH_DATA = { "armv7m": ArchData( "EM_ARM", MP_NATIVE_ARCH_ARMV7M << 2, - 2, 4, (R_ARM_GOT_BREL,), asm_jump_thumb2, @@ -156,7 +153,6 @@ ARCH_DATA = { "armv7emsp": ArchData( "EM_ARM", MP_NATIVE_ARCH_ARMV7EMSP << 2, - 2, 4, (R_ARM_GOT_BREL,), asm_jump_thumb2, @@ -164,7 +160,6 @@ ARCH_DATA = { "armv7emdp": ArchData( "EM_ARM", MP_NATIVE_ARCH_ARMV7EMDP << 2, - 2, 4, (R_ARM_GOT_BREL,), asm_jump_thumb2, @@ -172,7 +167,6 @@ ARCH_DATA = { "xtensa": ArchData( "EM_XTENSA", MP_NATIVE_ARCH_XTENSA << 2, - 2, 4, (R_XTENSA_32, R_XTENSA_PLT), asm_jump_xtensa, @@ -181,9 +175,9 @@ ARCH_DATA = { "EM_XTENSA", MP_NATIVE_ARCH_XTENSAWIN << 2, 4, - 4, (R_XTENSA_32, R_XTENSA_PLT), asm_jump_xtensa, + separate_rodata=True, ), } @@ -435,8 +429,8 @@ def populate_got(env): # Create a relocation for each GOT entry for got_entry in got_list: - if got_entry.name == "mp_fun_table": - dest = "mp_fun_table" + if got_entry.name in ("mp_native_qstr_table", "mp_native_obj_table", "mp_fun_table"): + dest = got_entry.name elif got_entry.name.startswith("mp_fun_table+0x"): dest = int(got_entry.name.split("+")[1], 16) // env.arch.word_size elif got_entry.sec_name.startswith(".text"): @@ -754,19 +748,19 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): env.lit_section = Section("LIT", bytearray(lit_size), env.arch.word_size) env.sections.insert(1, env.lit_section) - # Create section to contain mp_native_qstr_val_table - env.qstr_val_section = Section( - ".text.QSTR_VAL", + # Create section to contain mp_native_qstr_table + env.qstr_table_section = Section( + ".external.qstr_table", bytearray(native_qstr_vals_len * env.arch.qstr_entry_size), env.arch.qstr_entry_size, ) - env.sections.append(env.qstr_val_section) - # Create section to contain mp_native_qstr_obj_table - env.qstr_obj_section = Section( - ".text.QSTR_OBJ", bytearray(native_qstr_objs_len * env.arch.word_size), env.arch.word_size + # Create section to contain mp_native_obj_table + env.obj_table_section = Section( + ".external.obj_table", + bytearray(native_qstr_objs_len * env.arch.word_size), + env.arch.word_size, ) - env.sections.append(env.qstr_obj_section) # Resolve unknown symbols mp_fun_table_sec = Section(".external.mp_fun_table", b"", 0) @@ -796,10 +790,10 @@ def link_objects(env, native_qstr_vals_len, native_qstr_objs_len): pass elif sym.name == "mp_fun_table": sym.section = Section(".external", b"", 0) - elif sym.name == "mp_native_qstr_val_table": - sym.section = env.qstr_val_section - elif sym.name == "mp_native_qstr_obj_table": - sym.section = env.qstr_obj_section + elif sym.name == "mp_native_qstr_table": + sym.section = env.qstr_table_section + elif sym.name == "mp_native_obj_table": + sym.section = env.obj_table_section elif sym.name in env.known_syms: sym.resolved = env.known_syms[sym.name] else: @@ -923,13 +917,21 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) # MPY: n_qstr - out.write_uint(1) + out.write_uint(1 + len(native_qstr_vals)) # MPY: n_obj - out.write_uint(0) + out.write_uint(len(native_qstr_objs)) # MPY: qstr table out.write_qstr(fmpy) # filename + for q in native_qstr_vals: + out.write_qstr(q) + + # MPY: object table + for q in native_qstr_objs: + out.write_bytes(bytearray([MP_PERSISTENT_OBJ_STR])) + out.write_uint(len(q)) + out.write_bytes(bytes(q, "utf8") + b"\x00") # MPY: kind/len out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) @@ -937,17 +939,6 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): # MPY: machine code out.write_bytes(env.full_text) - # MPY: n_qstr_link (assumes little endian) - out.write_uint(len(native_qstr_vals) + len(native_qstr_objs)) - for q in range(len(native_qstr_vals)): - off = env.qstr_val_section.addr + q * env.arch.qstr_entry_size - out.write_uint(off << 2) - out.write_qstr(native_qstr_vals[q]) - for q in range(len(native_qstr_objs)): - off = env.qstr_obj_section.addr + q * env.arch.word_size - out.write_uint(off << 2 | 3) - out.write_qstr(native_qstr_objs[q]) - # MPY: scope_flags scope_flags = MP_SCOPE_FLAG_VIPERRELOC if len(env.full_rodata): @@ -978,10 +969,14 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): kind = 0 elif isinstance(kind, str) and kind.startswith(".bss"): kind = bss_const_table_idx - elif kind == "mp_fun_table": + elif kind == "mp_native_qstr_table": kind = 6 + elif kind == "mp_native_obj_table": + kind = 7 + elif kind == "mp_fun_table": + kind = 8 else: - kind = 7 + kind + kind = 9 + kind assert addr % env.arch.word_size == 0, addr offset = addr // env.arch.word_size if kind == prev_kind and base == prev_base and offset == prev_offset + 1: @@ -1023,18 +1018,14 @@ def do_preprocess(args): for i, q in enumerate(static_qstrs): print("#define %s (%u)" % (q, i + 1), file=f) for i, q in enumerate(sorted(qstr_vals)): - print("#define %s (mp_native_qstr_val_table[%d])" % (q, i), file=f) + print("#define %s (mp_native_qstr_table[%d])" % (q, i + 1), file=f) for i, q in enumerate(sorted(qstr_objs)): print( - "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_qstr_obj_table[%d])" % (q, i), + "#define MP_OBJ_NEW_QSTR_%s ((mp_obj_t)mp_native_obj_table[%d])" % (q, i), file=f, ) - if args.arch == "xtensawin": - qstr_type = "uint32_t" # esp32 can only read 32-bit values from IRAM - else: - qstr_type = "uint16_t" - print("extern const {} mp_native_qstr_val_table[];".format(qstr_type), file=f) - print("extern const mp_uint_t mp_native_qstr_obj_table[];", file=f) + print("extern const uint16_t mp_native_qstr_table[];", file=f) + print("extern const mp_uint_t mp_native_obj_table[];", file=f) def do_link(args):