djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

py/persistentcode: Introduce .mpy sub-version.

Browse Source 
The intent is to allow us to make breaking changes to the native ABI (e.g.
changes to dynruntime.h) without needing the bytecode version to increment.

With this commit the two bits previously used for the feature flags (but
now unused as of .mpy version 6) encode a sub-version.  A bytecode-only
.mpy file can be loaded as long as MPY_VERSION matches, but a native .mpy
(i.e. one with an arch set) must also match MPY_SUB_VERSION.  This allows 3
additional updates to the native ABI per bytecode revision.

The sub-version is set to 1 because the previous commits that changed the
layout of mp_obj_type_t have changed the native ABI.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
Signed-off-by: Damien George <damien@micropython.org>
This commit is contained in:
Jim Mussared 2022-09-17 23:57:12 +10:00 committed by Damien George
parent b41aaaa8a9
commit d94141e147
8 changed files with 46 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/reference/mpyfiles.rst
View File

@ -27,6 +27,11 @@ Compatibility is based on the following:
* Version of the .mpy file: the version of the file must match the version
supported by the system loading it.
* Sub-version of the .mpy file: if the .mpy file contains native machine code
then the sub-version of the file must match the version support by the
system loading it. Otherwise, if there is no native machine code in the .mpy
file, then the sub-version is ignored when loading.
* Small integer bits: the .mpy file will require a minimum number of bits in
a small integer and the system loading it must support at least this many
bits.
@ -55,6 +60,7 @@ If importing an .mpy file fails then try the following:
'armv6', 'armv6m', 'armv7m', 'armv7em', 'armv7emsp', 'armv7emdp',
'xtensa', 'xtensawin'][sys_mpy >> 10]
print('mpy version:', sys_mpy & 0xff)
print('mpy sub-version:', sys_mpy >> 8 & 3)
print('mpy flags:', end='')
if arch:
print(' -march=' + arch, end='')

2
mpy-cross/main.c
View File

@ -228,7 +228,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
a += 1;
} else if (strcmp(argv[a], "--version") == 0) {
printf("MicroPython " MICROPY_GIT_TAG " on " MICROPY_BUILD_DATE
"; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "\n");
"; mpy-cross emitting mpy v" MP_STRINGIFY(MPY_VERSION) "." MP_STRINGIFY(MPY_SUB_VERSION) "\n");
return 0;
} else if (strcmp(argv[a], "-v") == 0) {
mp_verbose_flag++;

6
py/persistentcode.c
View File

@ -393,14 +393,14 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) {
byte header[4];
read_bytes(reader, header, sizeof(header));
byte arch = MPY_FEATURE_DECODE_ARCH(header[2]);
if (header[0] != 'M'
|| header[1] != MPY_VERSION
|| MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS
|| (arch != MP_NATIVE_ARCH_NONE && MPY_FEATURE_DECODE_SUB_VERSION(header[2]) != MPY_SUB_VERSION)
|| header[3] > MP_SMALL_INT_BITS) {
mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
}
if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) {
byte arch = MPY_FEATURE_DECODE_ARCH(header[2]);
if (!MPY_FEATURE_ARCH_TEST(arch)) {
if (MPY_FEATURE_ARCH_TEST(MP_NATIVE_ARCH_NONE)) {
// On supported ports this can be resolved by enabling feature, eg
@ -596,7 +596,7 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) {
byte header[4] = {
'M',
MPY_VERSION,
MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS_DYNAMIC),
MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION),
#if MICROPY_DYNAMIC_COMPILER
mp_dynamic_compiler.small_int_bits,
#else

23
py/persistentcode.h
View File

@ -30,24 +30,23 @@
#include "py/reader.h"
#include "py/emitglue.h"
// The current version of .mpy files
// The current version of .mpy files. A bytecode-only .mpy file can be loaded
// as long as MPY_VERSION matches, but a native .mpy (i.e. one with an arch
// set) must also match MPY_SUB_VERSION. This allows 3 additional updates to
// the native ABI per bytecode revision.
#define MPY_VERSION 6
#define MPY_SUB_VERSION 1
// Macros to encode/decode flags to/from the feature byte
#define MPY_FEATURE_ENCODE_FLAGS(flags) (flags)
#define MPY_FEATURE_DECODE_FLAGS(feat) ((feat) & 3)
// Macros to encode/decode sub-version to/from the feature byte. This replaces
// the bits previously used to encode the flags (map caching and unicode)
// which are no longer used starting at .mpy version 6.
#define MPY_FEATURE_ENCODE_SUB_VERSION(version) (version)
#define MPY_FEATURE_DECODE_SUB_VERSION(feat) ((feat) & 3)
// Macros to encode/decode native architecture to/from the feature byte
#define MPY_FEATURE_ENCODE_ARCH(arch) ((arch) << 2)
#define MPY_FEATURE_DECODE_ARCH(feat) ((feat) >> 2)
// The feature flag bits encode the compile-time config options that affect
// the generate bytecode. Note: no longer used.
// (formerly MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE and MICROPY_PY_BUILTINS_STR_UNICODE).
#define MPY_FEATURE_FLAGS (0)
// This is a version of the flags that can be configured at runtime.
#define MPY_FEATURE_FLAGS_DYNAMIC (0)
// Define the host architecture
#if MICROPY_EMIT_X86
#define MPY_FEATURE_ARCH (MP_NATIVE_ARCH_X86)
@ -82,7 +81,7 @@
// 16-bit little-endian integer with the second and third bytes of supported .mpy files
#define MPY_FILE_HEADER_INT (MPY_VERSION \
| (MPY_FEATURE_ENCODE_FLAGS(MPY_FEATURE_FLAGS) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8)
| (MPY_FEATURE_ENCODE_SUB_VERSION(MPY_SUB_VERSION) | MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH)) << 8)
enum {
MP_NATIVE_ARCH_NONE = 0,

7
tests/micropython/import_mpy_native.py
View File

@ -11,7 +11,8 @@ except (ImportError, AttributeError):
raise SystemExit
mpy_arch = usys.implementation._mpy >> 8
if mpy_arch == 0:
if mpy_arch >> 2 == 0:
# This system does not support .mpy files containing native code
print("SKIP")
raise SystemExit
@ -54,8 +55,8 @@ class UserFS:
valid_header = bytes([77, 6, mpy_arch, 31])
# fmt: off
user_files = {
# bad architecture
'/mod0.mpy': b'M\x06\xfc\x1f',
# bad architecture (mpy_arch needed for sub-version)
'/mod0.mpy': bytes([77, 6, 0xfc | mpy_arch, 31]),
# test loading of viper and asm
'/mod1.mpy': valid_header + (

12
tests/micropython/import_mpy_native_gc.py
View File

@ -46,24 +46,26 @@ class UserFS:
# Pre-compiled examples/natmod/features0 example for various architectures, keyed
# by the required value of sys.implementation._mpy.
# by the required value of sys.implementation._mpy (without sub-version).
features0_file_contents = {
# -march=x64
0x806: b'M\x06\x08\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
0x806: b'M\x06\x09\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x8a\x02\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb7x\x02\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11$\r&\xa3 \x01"\xff',
# -march=armv6m
0x1006: b"M\x06\x10\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
0x1006: b"M\x06\x11\x1f\x02\x004build/features0.native.mpy\x00\x12factorial\x00\x88\x02\x18\xe0\x00\x00\x10\xb5\tK\tJ{D\x9cX\x02!\xe3h\x98G\x03\x00\x01 \x00+\x02\xd0XC\x01;\xfa\xe7\x02!#i\x98G\x10\xbd\xc0Fj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05\x00\x07K\x08I\xf3XyDX\x88ck\x98G(\x00\xb8G h\xf8\xbd\xc0F:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x11<\r>\xa38\x01:\xff",
}
# Populate armv7m-derived archs based on armv6m.
for arch in (0x1406, 0x1806, 0x1C06, 0x2006):
features0_file_contents[arch] = features0_file_contents[0x1006]
if sys.implementation._mpy not in features0_file_contents:
# Check that a .mpy exists for the target (ignore sub-version in lookup).
sys_implementation_mpy = sys.implementation._mpy & ~(3 << 8)
if sys_implementation_mpy not in features0_file_contents:
print("SKIP")
raise SystemExit
# These are the test .mpy files.
user_files = {"/features0.mpy": features0_file_contents[sys.implementation._mpy]}
user_files = {"/features0.mpy": features0_file_contents[sys_implementation_mpy]}
# Create and mount a user filesystem.
uos.mount(UserFS(user_files), "/userfs")

10
tools/mpy-tool.py
View File

@ -88,6 +88,7 @@ class FreezeError(Exception):
class Config:
MPY_VERSION = 6
MPY_SUB_VERSION = 1
MICROPY_LONGINT_IMPL_NONE = 0
MICROPY_LONGINT_IMPL_LONGLONG = 1
MICROPY_LONGINT_IMPL_MPZ = 2
@ -1335,6 +1336,9 @@ def read_mpy(filename):
feature_byte = header[2]
mpy_native_arch = feature_byte >> 2
if mpy_native_arch != MP_NATIVE_ARCH_NONE:
mpy_sub_version = feature_byte & 3
if mpy_sub_version != config.MPY_SUB_VERSION:
raise MPYReadError(filename, "incompatible .mpy sub-version")
if config.native_arch == MP_NATIVE_ARCH_NONE:
config.native_arch = mpy_native_arch
elif config.native_arch != mpy_native_arch:
@ -1658,7 +1662,9 @@ def merge_mpy(compiled_modules, output_file):
else:
main_cm_idx = None
for idx, cm in enumerate(compiled_modules):
if cm.header[2]:
feature_byte = cm.header[2]
mpy_native_arch = feature_byte >> 2
if mpy_native_arch:
# Must use qstr_table and obj_table from this raw_code
if main_cm_idx is not None:
raise Exception("can't merge files when more than one contains native code")
@ -1670,7 +1676,7 @@ def merge_mpy(compiled_modules, output_file):
header = bytearray(4)
header[0] = ord("M")
header[1] = config.MPY_VERSION
header[2] = config.native_arch << 2
header[2] = config.native_arch << 2 | config.MPY_SUB_VERSION
header[3] = config.mp_small_int_bits
merged_mpy.extend(header)

7
tools/mpy_ld.py
View File

@ -36,6 +36,7 @@ import makeqstrdata as qstrutil
# MicroPython constants
MPY_VERSION = 6
MPY_SUB_VERSION = 1
MP_CODE_BYTECODE = 2
MP_CODE_NATIVE_VIPER = 4
MP_NATIVE_ARCH_X86 = 1
@ -917,7 +918,11 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs):
out.open(fmpy)
# MPY: header
out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS]))
out.write_bytes(
bytearray(
[ord("M"), MPY_VERSION, env.arch.mpy_feature | MPY_SUB_VERSION, MP_SMALL_INT_BITS]
)
)
# MPY: n_qstr
out.write_uint(1 + len(native_qstr_vals))