diff --git a/esp8266/mpconfigport.h b/esp8266/mpconfigport.h index 7d5914191a..a19572a20d 100644 --- a/esp8266/mpconfigport.h +++ b/esp8266/mpconfigport.h @@ -61,7 +61,7 @@ #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_FLOAT) #define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_NORMAL) #define MICROPY_STREAMS_NON_BLOCK (1) -#define MICROPY_MODULE_FROZEN (1) +#define MICROPY_MODULE_FROZEN_STR (1) #define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str32 #define MICROPY_FATFS_ENABLE_LFN (1) diff --git a/lib/utils/pyexec.c b/lib/utils/pyexec.c index 8afa3813cd..a9e058c1c4 100644 --- a/lib/utils/pyexec.c +++ b/lib/utils/pyexec.c @@ -50,22 +50,33 @@ STATIC bool repl_display_debugging_info = 0; #define EXEC_FLAG_PRINT_EOF (1) #define EXEC_FLAG_ALLOW_DEBUGGING (2) #define EXEC_FLAG_IS_REPL (4) +#define EXEC_FLAG_SOURCE_IS_RAW_CODE (8) // parses, compiles and executes the code in the lexer // frees the lexer before returning // EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output // EXEC_FLAG_ALLOW_DEBUGGING allows debugging info to be printed after executing the code // EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile) -STATIC int parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, int exec_flags) { +STATIC int parse_compile_execute(void *source, mp_parse_input_kind_t input_kind, int exec_flags) { int ret = 0; uint32_t start = 0; nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - // parse and compile the script - qstr source_name = lex->source_name; - mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); - mp_obj_t module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL); + mp_obj_t module_fun; + #if MICROPY_MODULE_FROZEN_MPY + if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { + // source is a raw_code object, create the function + module_fun = mp_make_function_from_raw_code(source, MP_OBJ_NULL, MP_OBJ_NULL); + } else + #endif + { + // source is a lexer, parse and compile the script + mp_lexer_t *lex = source; + qstr source_name = lex->source_name; + mp_parse_tree_t parse_tree = mp_parse(lex, input_kind); + module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL); + } // execute code mp_hal_set_interrupt_char(CHAR_CTRL_C); // allow ctrl-C to interrupt us @@ -488,14 +499,24 @@ int pyexec_file(const char *filename) { #if MICROPY_MODULE_FROZEN int pyexec_frozen_module(const char *name) { - mp_lexer_t *lex = mp_find_frozen_module(name, strlen(name)); + void *frozen_data; + int frozen_type = mp_find_frozen_module(name, strlen(name), &frozen_data); - if (lex == NULL) { - printf("could not find module '%s'\n", name); - return false; + switch (frozen_type) { + #if MICROPY_MODULE_FROZEN_STR + case MP_FROZEN_STR: + return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, 0); + #endif + + #if MICROPY_MODULE_FROZEN_MPY + case MP_FROZEN_MPY: + return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_RAW_CODE); + #endif + + default: + printf("could not find module '%s'\n", name); + return false; } - - return parse_compile_execute(lex, MP_PARSE_FILE_INPUT, 0); } #endif diff --git a/pic16bit/mpconfigport.h b/pic16bit/mpconfigport.h index af30296935..7335ecf348 100644 --- a/pic16bit/mpconfigport.h +++ b/pic16bit/mpconfigport.h @@ -60,7 +60,6 @@ #define MICROPY_PY_IO (0) #define MICROPY_PY_STRUCT (0) #define MICROPY_PY_SYS (0) -#define MICROPY_MODULE_FROZEN (0) #define MICROPY_CPYTHON_COMPAT (0) #define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ) #define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_NONE) diff --git a/py/builtinimport.c b/py/builtinimport.c index ec79357cb7..0e4dce6430 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -144,7 +144,7 @@ STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex, const char } #endif -#if MICROPY_PERSISTENT_CODE_LOAD +#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_MODULE_FROZEN_MPY STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) { #if MICROPY_PY___FILE__ // TODO @@ -182,8 +182,9 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) { #endif STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { - // create the lexer + #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_ENABLE_COMPILER char *file_str = vstr_null_terminated_str(file); + #endif #if MICROPY_PERSISTENT_CODE_LOAD if (file_str[file->len - 3] == 'm') { @@ -340,8 +341,9 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { DEBUG_printf("Module not yet loaded\n"); #if MICROPY_MODULE_FROZEN - mp_lexer_t *lex = mp_find_frozen_module(mod_str, mod_len); - if (lex != NULL) { + void *frozen_data; + int frozen_type = mp_find_frozen_module(mod_str, mod_len, &frozen_data); + if (frozen_type != MP_FROZEN_NONE) { module_obj = mp_obj_new_module(module_name_qstr); // if args[3] (fromtuple) has magic value False, set up // this module for command-line "-m" option (set module's @@ -351,7 +353,16 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj); mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__)); } - do_load_from_lexer(module_obj, lex, mod_str); + #if MICROPY_MODULE_FROZEN_STR + if (frozen_type == MP_FROZEN_STR) { + do_load_from_lexer(module_obj, frozen_data, mod_str); + } + #endif + #if MICROPY_MODULE_FROZEN_MPY + if (frozen_type == MP_FROZEN_MPY) { + do_execute_raw_code(module_obj, frozen_data); + } + #endif return module_obj; } #endif diff --git a/py/emitglue.c b/py/emitglue.c index 7e5edce756..c657d40d12 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -172,7 +172,7 @@ mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_clos return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2)); } -#if MICROPY_PERSISTENT_CODE +#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #include "py/smallint.h" @@ -229,7 +229,7 @@ STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_ } } -#endif // MICROPY_PERSISTENT_CODE +#endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #if MICROPY_PERSISTENT_CODE_LOAD diff --git a/py/frozenmod.c b/py/frozenmod.c index 6b76bf662b..92afdb9a5e 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -4,6 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2015 Paul Sokolovsky + * Copyright (c) 2016 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,7 +31,7 @@ #include "py/lexer.h" #include "py/frozenmod.h" -#if MICROPY_MODULE_FROZEN +#if MICROPY_MODULE_FROZEN_STR #ifndef MICROPY_MODULE_FROZEN_LEXER #define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str_len @@ -38,24 +39,67 @@ mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len); #endif -extern const char mp_frozen_names[]; -extern const uint32_t mp_frozen_sizes[]; -extern const char mp_frozen_content[]; +extern const char mp_frozen_str_names[]; +extern const uint32_t mp_frozen_str_sizes[]; +extern const char mp_frozen_str_content[]; -mp_lexer_t *mp_find_frozen_module(const char *str, int len) { - const char *name = mp_frozen_names; +STATIC mp_lexer_t *mp_find_frozen_str(const char *str, size_t len) { + const char *name = mp_frozen_str_names; size_t offset = 0; for (int i = 0; *name != 0; i++) { - int l = strlen(name); + size_t l = strlen(name); if (l == len && !memcmp(str, name, l)) { - mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_content + offset, mp_frozen_sizes[i], 0); + mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_str_content + offset, mp_frozen_str_sizes[i], 0); return lex; } name += l + 1; - offset += mp_frozen_sizes[i] + 1; + offset += mp_frozen_str_sizes[i] + 1; } return NULL; } -#endif // MICROPY_MODULE_FROZEN +#endif + +#if MICROPY_MODULE_FROZEN_MPY + +#include "py/emitglue.h" + +extern const char mp_frozen_mpy_names[]; +extern const mp_raw_code_t *const mp_frozen_mpy_content[]; + +STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t len) { + const char *name = mp_frozen_mpy_names; + for (size_t i = 0; *name != 0; i++) { + size_t l = strlen(name); + if (l == len && !memcmp(str, name, l)) { + return mp_frozen_mpy_content[i]; + } + name += l + 1; + } + return NULL; +} + +#endif + +#if MICROPY_MODULE_FROZEN + +int mp_find_frozen_module(const char *str, size_t len, void **data) { + #if MICROPY_MODULE_FROZEN_STR + mp_lexer_t *lex = mp_find_frozen_str(str, len); + if (lex != NULL) { + *data = lex; + return MP_FROZEN_STR; + } + #endif + #if MICROPY_MODULE_FROZEN_MPY + const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len); + if (rc != NULL) { + *data = (void*)rc; + return MP_FROZEN_MPY; + } + #endif + return MP_FROZEN_NONE; +} + +#endif diff --git a/py/frozenmod.h b/py/frozenmod.h index 67caced14d..a1638d2293 100644 --- a/py/frozenmod.h +++ b/py/frozenmod.h @@ -24,4 +24,10 @@ * THE SOFTWARE. */ -mp_lexer_t *mp_find_frozen_module(const char *str, int len); +enum { + MP_FROZEN_NONE, + MP_FROZEN_STR, + MP_FROZEN_MPY, +}; + +int mp_find_frozen_module(const char *str, size_t len, void **data); diff --git a/py/mpconfig.h b/py/mpconfig.h index 77ac614939..42ef19b72c 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -234,7 +234,7 @@ // Whether generated code can persist independently of the VM/runtime instance // This is enabled automatically when needed by other features #ifndef MICROPY_PERSISTENT_CODE -#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE) +#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY) #endif // Whether to emit x64 native code @@ -526,9 +526,19 @@ typedef double mp_float_t; #define MICROPY_MODULE_WEAK_LINKS (0) #endif -// Whether frozen modules are supported +// Whether frozen modules are supported in the form of strings +#ifndef MICROPY_MODULE_FROZEN_STR +#define MICROPY_MODULE_FROZEN_STR (0) +#endif + +// Whether frozen modules are supported in the form of .mpy files +#ifndef MICROPY_MODULE_FROZEN_MPY +#define MICROPY_MODULE_FROZEN_MPY (0) +#endif + +// Convenience macro for whether frozen modules are supported #ifndef MICROPY_MODULE_FROZEN -#define MICROPY_MODULE_FROZEN (0) +#define MICROPY_MODULE_FROZEN (MICROPY_MODULE_FROZEN_STR || MICROPY_MODULE_FROZEN_MPY) #endif // Whether you can override builtins in the builtins module diff --git a/py/qstr.c b/py/qstr.c index 4268946fbe..ef31a682ee 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -87,11 +87,11 @@ mp_uint_t qstr_compute_hash(const byte *data, size_t len) { return hash; } -STATIC const qstr_pool_t const_pool = { +const qstr_pool_t mp_qstr_const_pool = { NULL, // no previous pool 0, // no previous pool 10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below) - MP_QSTR_number_of, // corresponds to number of strings in array just below + MP_QSTRnumber_of, // corresponds to number of strings in array just below { #define QDEF(id, str) str, #include "genhdr/qstrdefs.generated.h" @@ -99,8 +99,15 @@ STATIC const qstr_pool_t const_pool = { }, }; +#ifdef MICROPY_QSTR_EXTRA_POOL +extern const qstr_pool_t MICROPY_QSTR_EXTRA_POOL; +#define CONST_POOL MICROPY_QSTR_EXTRA_POOL +#else +#define CONST_POOL mp_qstr_const_pool +#endif + void qstr_init(void) { - MP_STATE_VM(last_pool) = (qstr_pool_t*)&const_pool; // we won't modify the const_pool since it has no allocated room left + MP_STATE_VM(last_pool) = (qstr_pool_t*)&CONST_POOL; // we won't modify the const_pool since it has no allocated room left MP_STATE_VM(qstr_last_chunk) = NULL; } @@ -258,7 +265,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si *n_qstr = 0; *n_str_data_bytes = 0; *n_total_bytes = 0; - for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) { + for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) { *n_pool += 1; *n_qstr += pool->len; for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) { @@ -275,7 +282,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si #if MICROPY_PY_MICROPYTHON_MEM_INFO void qstr_dump_data(void) { - for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) { + for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) { for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) { mp_printf(&mp_plat_print, "Q(%s)\n", Q_GET_DATA(*q)); } diff --git a/py/qstr.h b/py/qstr.h index b68f60a443..b5c261f0d5 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -40,7 +40,7 @@ enum { #define QDEF(id, str) id, #include "genhdr/qstrdefs.generated.h" #undef QDEF - MP_QSTR_number_of, + MP_QSTRnumber_of, // no underscore so it can't clash with any of the above }; typedef size_t qstr; diff --git a/stmhal/Makefile b/stmhal/Makefile index 847cb4e8b2..7dde98bd14 100644 --- a/stmhal/Makefile +++ b/stmhal/Makefile @@ -267,7 +267,7 @@ $(BUILD)/$(HAL_DIR)/src/stm32$(MCU_SERIES)xx_hal_sd.o: COPT += -Os all: $(BUILD)/firmware.dfu $(BUILD)/firmware.hex ifneq ($(FROZEN_DIR),) -CFLAGS += -DMICROPY_MODULE_FROZEN +CFLAGS += -DMICROPY_MODULE_FROZEN_STR OBJ += $(BUILD)/frozen-files.o MAKE_FROZEN = ../tools/make-frozen.py diff --git a/teensy/Makefile b/teensy/Makefile index c182631802..5a03f28eef 100644 --- a/teensy/Makefile +++ b/teensy/Makefile @@ -154,7 +154,7 @@ endif # USE_MEMZIP ifeq ($(USE_FROZEN),1) -CFLAGS += -DMICROPY_MODULE_FROZEN +CFLAGS += -DMICROPY_MODULE_FROZEN_STR SRC_C += \ lexerfrozen.c diff --git a/tools/make-frozen.py b/tools/make-frozen.py index e0c807c4ef..84e589b985 100755 --- a/tools/make-frozen.py +++ b/tools/make-frozen.py @@ -14,7 +14,7 @@ # # ./make-frozen.py frozen > frozen.c # -# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN in +# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN_STR in # config. # from __future__ import print_function @@ -37,20 +37,20 @@ for dirpath, dirnames, filenames in os.walk(root): modules.append((fullpath[root_len + 1:], st)) print("#include ") -print("const char mp_frozen_names[] = {") +print("const char mp_frozen_str_names[] = {") for f, st in modules: m = module_name(f) print('"%s\\0"' % m) print('"\\0"};') -print("const uint32_t mp_frozen_sizes[] = {") +print("const uint32_t mp_frozen_str_sizes[] = {") for f, st in modules: print("%d," % st.st_size) print("};") -print("const char mp_frozen_content[] = {") +print("const char mp_frozen_str_content[] = {") for f, st in modules: data = open(sys.argv[1] + "/" + f, "rb").read() # Python2 vs Python3 tricks diff --git a/unix/mpconfigport.h b/unix/mpconfigport.h index 1a6eb6f8d1..28a7e8b763 100644 --- a/unix/mpconfigport.h +++ b/unix/mpconfigport.h @@ -94,7 +94,7 @@ #define MICROPY_PY_CMATH (1) #define MICROPY_PY_IO_FILEIO (1) #define MICROPY_PY_GC_COLLECT_RETVAL (1) -#define MICROPY_MODULE_FROZEN (1) +#define MICROPY_MODULE_FROZEN_STR (1) #define MICROPY_STACKLESS (0) #define MICROPY_STACKLESS_STRICT (0) diff --git a/unix/mpconfigport_fast.h b/unix/mpconfigport_fast.h index 0694b7099c..b5be9f334f 100644 --- a/unix/mpconfigport_fast.h +++ b/unix/mpconfigport_fast.h @@ -36,5 +36,5 @@ // Don't include builtin upip, as this build is again intended just for // synthetic benchmarking -#undef MICROPY_MODULE_FROZEN -#define MICROPY_MODULE_FROZEN (0) +#undef MICROPY_MODULE_FROZEN_STR +#define MICROPY_MODULE_FROZEN_STR (0) diff --git a/windows/mpconfigport.h b/windows/mpconfigport.h index ad79ef3813..203aa04494 100644 --- a/windows/mpconfigport.h +++ b/windows/mpconfigport.h @@ -77,7 +77,7 @@ #define MICROPY_PY_CMATH (1) #define MICROPY_PY_IO_FILEIO (1) #define MICROPY_PY_GC_COLLECT_RETVAL (1) -#define MICROPY_MODULE_FROZEN (0) +#define MICROPY_MODULE_FROZEN_STR (0) #define MICROPY_STACKLESS (0) #define MICROPY_STACKLESS_STRICT (0)