djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

py: Add ability to have frozen persistent bytecode from .mpy files.

Browse Source 
The config variable MICROPY_MODULE_FROZEN is now made of two separate
parts: MICROPY_MODULE_FROZEN_STR and MICROPY_MODULE_FROZEN_MPY.  This
allows to have none, either or both of frozen strings and frozen mpy
files (aka frozen bytecode).
This commit is contained in:
Damien George 2016-01-31 22:24:16 +00:00
parent 0699c6bf9e
commit 0a2e9650f5
16 changed files with 148 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
esp8266/mpconfigport.h
View File

@ -61,7 +61,7 @@
#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_FLOAT)
#define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_NORMAL)
#define MICROPY_STREAMS_NON_BLOCK (1)
#define MICROPY_MODULE_FROZEN (1)
#define MICROPY_MODULE_FROZEN_STR (1)
#define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str32
#define MICROPY_FATFS_ENABLE_LFN (1)

43
lib/utils/pyexec.c
View File

@ -50,22 +50,33 @@ STATIC bool repl_display_debugging_info = 0;
#define EXEC_FLAG_PRINT_EOF (1)
#define EXEC_FLAG_ALLOW_DEBUGGING (2)
#define EXEC_FLAG_IS_REPL (4)
#define EXEC_FLAG_SOURCE_IS_RAW_CODE (8)
// parses, compiles and executes the code in the lexer
// frees the lexer before returning
// EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output
// EXEC_FLAG_ALLOW_DEBUGGING allows debugging info to be printed after executing the code
// EXEC_FLAG_IS_REPL is used for REPL inputs (flag passed on to mp_compile)
STATIC int parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, int exec_flags) {
STATIC int parse_compile_execute(void *source, mp_parse_input_kind_t input_kind, int exec_flags) {
int ret = 0;
uint32_t start = 0;
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
// parse and compile the script
qstr source_name = lex->source_name;
mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
mp_obj_t module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
mp_obj_t module_fun;
#if MICROPY_MODULE_FROZEN_MPY
if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) {
// source is a raw_code object, create the function
module_fun = mp_make_function_from_raw_code(source, MP_OBJ_NULL, MP_OBJ_NULL);
} else
#endif
{
// source is a lexer, parse and compile the script
mp_lexer_t *lex = source;
qstr source_name = lex->source_name;
mp_parse_tree_t parse_tree = mp_parse(lex, input_kind);
module_fun = mp_compile(&parse_tree, source_name, MP_EMIT_OPT_NONE, exec_flags & EXEC_FLAG_IS_REPL);
}
// execute code
mp_hal_set_interrupt_char(CHAR_CTRL_C); // allow ctrl-C to interrupt us
@ -488,14 +499,24 @@ int pyexec_file(const char *filename) {
#if MICROPY_MODULE_FROZEN
int pyexec_frozen_module(const char *name) {
mp_lexer_t *lex = mp_find_frozen_module(name, strlen(name));
void *frozen_data;
int frozen_type = mp_find_frozen_module(name, strlen(name), &frozen_data);
if (lex == NULL) {
printf("could not find module '%s'\n", name);
return false;
switch (frozen_type) {
#if MICROPY_MODULE_FROZEN_STR
case MP_FROZEN_STR:
return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, 0);
#endif
#if MICROPY_MODULE_FROZEN_MPY
case MP_FROZEN_MPY:
return parse_compile_execute(frozen_data, MP_PARSE_FILE_INPUT, EXEC_FLAG_SOURCE_IS_RAW_CODE);
#endif
default:
printf("could not find module '%s'\n", name);
return false;
}
return parse_compile_execute(lex, MP_PARSE_FILE_INPUT, 0);
}
#endif

1
pic16bit/mpconfigport.h
View File

@ -60,7 +60,6 @@
#define MICROPY_PY_IO (0)
#define MICROPY_PY_STRUCT (0)
#define MICROPY_PY_SYS (0)
#define MICROPY_MODULE_FROZEN (0)
#define MICROPY_CPYTHON_COMPAT (0)
#define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ)
#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_NONE)

21
py/builtinimport.c
View File

@ -144,7 +144,7 @@ STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex, const char
}
#endif
#if MICROPY_PERSISTENT_CODE_LOAD
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_MODULE_FROZEN_MPY
STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
#if MICROPY_PY___FILE__
// TODO
@ -182,8 +182,9 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
#endif
STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
// create the lexer
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_ENABLE_COMPILER
char *file_str = vstr_null_terminated_str(file);
#endif
#if MICROPY_PERSISTENT_CODE_LOAD
if (file_str[file->len - 3] == 'm') {
@ -340,8 +341,9 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
DEBUG_printf("Module not yet loaded\n");
#if MICROPY_MODULE_FROZEN
mp_lexer_t *lex = mp_find_frozen_module(mod_str, mod_len);
if (lex != NULL) {
void *frozen_data;
int frozen_type = mp_find_frozen_module(mod_str, mod_len, &frozen_data);
if (frozen_type != MP_FROZEN_NONE) {
module_obj = mp_obj_new_module(module_name_qstr);
// if args[3] (fromtuple) has magic value False, set up
// this module for command-line "-m" option (set module's
@ -351,7 +353,16 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj);
mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__));
}
do_load_from_lexer(module_obj, lex, mod_str);
#if MICROPY_MODULE_FROZEN_STR
if (frozen_type == MP_FROZEN_STR) {
do_load_from_lexer(module_obj, frozen_data, mod_str);
}
#endif
#if MICROPY_MODULE_FROZEN_MPY
if (frozen_type == MP_FROZEN_MPY) {
do_execute_raw_code(module_obj, frozen_data);
}
#endif
return module_obj;
}
#endif

4
py/emitglue.c
View File

@ -172,7 +172,7 @@ mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_clos
return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2));
}
#if MICROPY_PERSISTENT_CODE
#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
#include "py/smallint.h"
@ -229,7 +229,7 @@ STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_
}
}
#endif // MICROPY_PERSISTENT_CODE
#endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
#if MICROPY_PERSISTENT_CODE_LOAD

64
py/frozenmod.c
View File

@ -4,6 +4,7 @@
* The MIT License (MIT)
*
* Copyright (c) 2015 Paul Sokolovsky
* Copyright (c) 2016 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@ -30,7 +31,7 @@
#include "py/lexer.h"
#include "py/frozenmod.h"
#if MICROPY_MODULE_FROZEN
#if MICROPY_MODULE_FROZEN_STR
#ifndef MICROPY_MODULE_FROZEN_LEXER
#define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str_len
@ -38,24 +39,67 @@
mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len);
#endif
extern const char mp_frozen_names[];
extern const uint32_t mp_frozen_sizes[];
extern const char mp_frozen_content[];
extern const char mp_frozen_str_names[];
extern const uint32_t mp_frozen_str_sizes[];
extern const char mp_frozen_str_content[];
mp_lexer_t *mp_find_frozen_module(const char *str, int len) {
const char *name = mp_frozen_names;
STATIC mp_lexer_t *mp_find_frozen_str(const char *str, size_t len) {
const char *name = mp_frozen_str_names;
size_t offset = 0;
for (int i = 0; *name != 0; i++) {
int l = strlen(name);
size_t l = strlen(name);
if (l == len && !memcmp(str, name, l)) {
mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_content + offset, mp_frozen_sizes[i], 0);
mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_str_content + offset, mp_frozen_str_sizes[i], 0);
return lex;
}
name += l + 1;
offset += mp_frozen_sizes[i] + 1;
offset += mp_frozen_str_sizes[i] + 1;
}
return NULL;
}
#endif // MICROPY_MODULE_FROZEN
#endif
#if MICROPY_MODULE_FROZEN_MPY
#include "py/emitglue.h"
extern const char mp_frozen_mpy_names[];
extern const mp_raw_code_t *const mp_frozen_mpy_content[];
STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t len) {
const char *name = mp_frozen_mpy_names;
for (size_t i = 0; *name != 0; i++) {
size_t l = strlen(name);
if (l == len && !memcmp(str, name, l)) {
return mp_frozen_mpy_content[i];
}
name += l + 1;
}
return NULL;
}
#endif
#if MICROPY_MODULE_FROZEN
int mp_find_frozen_module(const char *str, size_t len, void **data) {
#if MICROPY_MODULE_FROZEN_STR
mp_lexer_t *lex = mp_find_frozen_str(str, len);
if (lex != NULL) {
*data = lex;
return MP_FROZEN_STR;
}
#endif
#if MICROPY_MODULE_FROZEN_MPY
const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len);
if (rc != NULL) {
*data = (void*)rc;
return MP_FROZEN_MPY;
}
#endif
return MP_FROZEN_NONE;
}
#endif

8
py/frozenmod.h
View File

@ -24,4 +24,10 @@
* THE SOFTWARE.
*/
mp_lexer_t *mp_find_frozen_module(const char *str, int len);
enum {
MP_FROZEN_NONE,
MP_FROZEN_STR,
MP_FROZEN_MPY,
};
int mp_find_frozen_module(const char *str, size_t len, void **data);

16
py/mpconfig.h
View File

@ -234,7 +234,7 @@
// Whether generated code can persist independently of the VM/runtime instance
// This is enabled automatically when needed by other features
#ifndef MICROPY_PERSISTENT_CODE
#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE)
#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY)
#endif
// Whether to emit x64 native code
@ -526,9 +526,19 @@ typedef double mp_float_t;
#define MICROPY_MODULE_WEAK_LINKS (0)
#endif
// Whether frozen modules are supported
// Whether frozen modules are supported in the form of strings
#ifndef MICROPY_MODULE_FROZEN_STR
#define MICROPY_MODULE_FROZEN_STR (0)
#endif
// Whether frozen modules are supported in the form of .mpy files
#ifndef MICROPY_MODULE_FROZEN_MPY
#define MICROPY_MODULE_FROZEN_MPY (0)
#endif
// Convenience macro for whether frozen modules are supported
#ifndef MICROPY_MODULE_FROZEN
#define MICROPY_MODULE_FROZEN (0)
#define MICROPY_MODULE_FROZEN (MICROPY_MODULE_FROZEN_STR || MICROPY_MODULE_FROZEN_MPY)
#endif
// Whether you can override builtins in the builtins module

17
py/qstr.c
View File

@ -87,11 +87,11 @@ mp_uint_t qstr_compute_hash(const byte *data, size_t len) {
return hash;
}
STATIC const qstr_pool_t const_pool = {
const qstr_pool_t mp_qstr_const_pool = {
NULL, // no previous pool
0, // no previous pool
10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
MP_QSTR_number_of, // corresponds to number of strings in array just below
MP_QSTRnumber_of, // corresponds to number of strings in array just below
{
#define QDEF(id, str) str,
#include "genhdr/qstrdefs.generated.h"
@ -99,8 +99,15 @@ STATIC const qstr_pool_t const_pool = {
},
};
#ifdef MICROPY_QSTR_EXTRA_POOL
extern const qstr_pool_t MICROPY_QSTR_EXTRA_POOL;
#define CONST_POOL MICROPY_QSTR_EXTRA_POOL
#else
#define CONST_POOL mp_qstr_const_pool
#endif
void qstr_init(void) {
MP_STATE_VM(last_pool) = (qstr_pool_t*)&const_pool; // we won't modify the const_pool since it has no allocated room left
MP_STATE_VM(last_pool) = (qstr_pool_t*)&CONST_POOL; // we won't modify the const_pool since it has no allocated room left
MP_STATE_VM(qstr_last_chunk) = NULL;
}
@ -258,7 +265,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
*n_qstr = 0;
*n_str_data_bytes = 0;
*n_total_bytes = 0;
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) {
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
*n_pool += 1;
*n_qstr += pool->len;
for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
@ -275,7 +282,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
#if MICROPY_PY_MICROPYTHON_MEM_INFO
void qstr_dump_data(void) {
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) {
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
mp_printf(&mp_plat_print, "Q(%s)\n", Q_GET_DATA(*q));
}

2
py/qstr.h
View File

@ -40,7 +40,7 @@ enum {
#define QDEF(id, str) id,
#include "genhdr/qstrdefs.generated.h"
#undef QDEF
MP_QSTR_number_of,
MP_QSTRnumber_of, // no underscore so it can't clash with any of the above
};
typedef size_t qstr;

2
stmhal/Makefile
View File

@ -267,7 +267,7 @@ $(BUILD)/$(HAL_DIR)/src/stm32$(MCU_SERIES)xx_hal_sd.o: COPT += -Os
all: $(BUILD)/firmware.dfu $(BUILD)/firmware.hex
ifneq ($(FROZEN_DIR),)
CFLAGS += -DMICROPY_MODULE_FROZEN
CFLAGS += -DMICROPY_MODULE_FROZEN_STR
OBJ += $(BUILD)/frozen-files.o
MAKE_FROZEN = ../tools/make-frozen.py

2
teensy/Makefile
View File

@ -154,7 +154,7 @@ endif # USE_MEMZIP
ifeq ($(USE_FROZEN),1)
CFLAGS += -DMICROPY_MODULE_FROZEN
CFLAGS += -DMICROPY_MODULE_FROZEN_STR
SRC_C += \
lexerfrozen.c

8
tools/make-frozen.py
View File

@ -14,7 +14,7 @@
#
# ./make-frozen.py frozen > frozen.c
#
# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN in
# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN_STR in
# config.
#
from __future__ import print_function
@ -37,20 +37,20 @@ for dirpath, dirnames, filenames in os.walk(root):
modules.append((fullpath[root_len + 1:], st))
print("#include <stdint.h>")
print("const char mp_frozen_names[] = {")
print("const char mp_frozen_str_names[] = {")
for f, st in modules:
m = module_name(f)
print('"%s\\0"' % m)
print('"\\0"};')
print("const uint32_t mp_frozen_sizes[] = {")
print("const uint32_t mp_frozen_str_sizes[] = {")
for f, st in modules:
print("%d," % st.st_size)
print("};")
print("const char mp_frozen_content[] = {")
print("const char mp_frozen_str_content[] = {")
for f, st in modules:
data = open(sys.argv[1] + "/" + f, "rb").read()
# Python2 vs Python3 tricks

2
unix/mpconfigport.h
View File

@ -94,7 +94,7 @@
#define MICROPY_PY_CMATH (1)
#define MICROPY_PY_IO_FILEIO (1)
#define MICROPY_PY_GC_COLLECT_RETVAL (1)
#define MICROPY_MODULE_FROZEN (1)
#define MICROPY_MODULE_FROZEN_STR (1)
#define MICROPY_STACKLESS (0)
#define MICROPY_STACKLESS_STRICT (0)

4
unix/mpconfigport_fast.h
View File

@ -36,5 +36,5 @@
// Don't include builtin upip, as this build is again intended just for
// synthetic benchmarking
#undef MICROPY_MODULE_FROZEN
#define MICROPY_MODULE_FROZEN (0)
#undef MICROPY_MODULE_FROZEN_STR
#define MICROPY_MODULE_FROZEN_STR (0)

2
windows/mpconfigport.h
View File

@ -77,7 +77,7 @@
#define MICROPY_PY_CMATH (1)
#define MICROPY_PY_IO_FILEIO (1)
#define MICROPY_PY_GC_COLLECT_RETVAL (1)
#define MICROPY_MODULE_FROZEN (0)
#define MICROPY_MODULE_FROZEN_STR (0)
#define MICROPY_STACKLESS (0)
#define MICROPY_STACKLESS_STRICT (0)