djsundog
/
circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mpy-cross: Add new component, a cross compiler for MicroPython bytecode. 

This component allows to generate .mpy files (pre compiled bytecode)
which can be executed within any MicroPython runtime/VM.
This commit is contained in:
Damien George 2016-02-11 22:37:26 +00:00
parent ea23520403
commit 56f76b873a
8 changed files with 632 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mpy-cross/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
mpy-cross

58
mpy-cross/Makefile Normal file
View File

@ -0,0 +1,58 @@
MICROPY_FORCE_32BIT = 1
include ../py/mkenv.mk
# define main target
PROG = mpy-cross
# qstr definitions (must come before including py.mk)
QSTR_DEFS = qstrdefsport.h
# OS name, for simple autoconfig
UNAME_S := $(shell uname -s)
# include py core make definitions
include ../py/py.mk
INC += -I.
INC += -I..
INC += -I$(BUILD)
# compiler settings
CWARN = -Wall -Werror
CWARN += -Wpointer-arith -Wuninitialized
CFLAGS = $(INC) $(CWARN) -ansi -std=gnu99 $(CFLAGS_MOD) $(COPT) $(CFLAGS_EXTRA)
CFLAGS += -fdata-sections -ffunction-sections -fno-asynchronous-unwind-tables
# Debugging/Optimization
ifdef DEBUG
CFLAGS += -g
COPT = -O0
else
COPT = -Os #-DNDEBUG
endif
# On OSX, 'gcc' is a symlink to clang unless a real gcc is installed.
# The unix port of micropython on OSX must be compiled with clang,
# while cross-compile ports require gcc, so we test here for OSX and
# if necessary override the value of 'CC' set in py/mkenv.mk
ifeq ($(UNAME_S),Darwin)
CC = clang
# Use clang syntax for map file
LDFLAGS_ARCH = -Wl,-map,$@.map
else
# Use gcc syntax for map file
LDFLAGS_ARCH = -Wl,-Map=$@.map,--cref
endif
LDFLAGS = $(LDFLAGS_MOD) $(LDFLAGS_ARCH) -lm $(LDFLAGS_EXTRA)
LDFLAGS += -Wl,--gc-sections
# source files
SRC_C = \
main.c \
gccollect.c \
OBJ = $(PY_O)
OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o))
include ../py/mkrules.mk

25
mpy-cross/README.md Normal file
View File

@ -0,0 +1,25 @@
MicroPython cross compiler
==========================
This directory contains the MicroPython cross compiler, which runs under any
Unix-like system and compiles .py scripts into .mpy files.
Build it as usual:
$ make
The compiler is called `mpy-cross`. Invoke it as:
$ ./mpy-cross foo.py
This will create a file foo.mpy which can then be copied to a place accessible
by the target MicroPython runtime (eg onto a pyboard's filesystem), and then
imported like any other Python module using `import foo`.
Different target runtimes may require a different format of the compiled
bytecode, and such options can be passed to the cross compiler. For example,
the unix port of MicroPython requires the following:
$ ./mpy-cross -mcache-lookup-bc foo.py
Run `./mpy-cross -h` to get a full list of options.

152
mpy-cross/gccollect.c Normal file
View File

@ -0,0 +1,152 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013-2014 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdio.h>
#include "py/mpstate.h"
#include "py/gc.h"
#if MICROPY_ENABLE_GC
// Even if we have specific support for an architecture, it is
// possible to force use of setjmp-based implementation.
#if !MICROPY_GCREGS_SETJMP
// We capture here callee-save registers, i.e. ones which may contain
// interesting values held there by our callers. It doesn't make sense
// to capture caller-saved registers, because they, well, put on the
// stack already by the caller.
#if defined(__x86_64__)
typedef mp_uint_t regs_t[6];
STATIC void gc_helper_get_regs(regs_t arr) {
register long rbx asm ("rbx");
register long rbp asm ("rbp");
register long r12 asm ("r12");
register long r13 asm ("r13");
register long r14 asm ("r14");
register long r15 asm ("r15");
#ifdef __clang__
// TODO:
// This is dirty workaround for Clang. It tries to get around
// uncompliant (wrt to GCC) behavior of handling register variables.
// Application of this patch here is random, and done only to unbreak
// MacOS build. Better, cross-arch ways to deal with Clang issues should
// be found.
asm("" : "=r"(rbx));
asm("" : "=r"(rbp));
asm("" : "=r"(r12));
asm("" : "=r"(r13));
asm("" : "=r"(r14));
asm("" : "=r"(r15));
#endif
arr[0] = rbx;
arr[1] = rbp;
arr[2] = r12;
arr[3] = r13;
arr[4] = r14;
arr[5] = r15;
}
#elif defined(__i386__)
typedef mp_uint_t regs_t[4];
STATIC void gc_helper_get_regs(regs_t arr) {
register long ebx asm ("ebx");
register long esi asm ("esi");
register long edi asm ("edi");
register long ebp asm ("ebp");
arr[0] = ebx;
arr[1] = esi;
arr[2] = edi;
arr[3] = ebp;
}
#elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__)
typedef mp_uint_t regs_t[10];
STATIC void gc_helper_get_regs(regs_t arr) {
register long r4 asm ("r4");
register long r5 asm ("r5");
register long r6 asm ("r6");
register long r7 asm ("r7");
register long r8 asm ("r8");
register long r9 asm ("r9");
register long r10 asm ("r10");
register long r11 asm ("r11");
register long r12 asm ("r12");
register long r13 asm ("r13");
arr[0] = r4;
arr[1] = r5;
arr[2] = r6;
arr[3] = r7;
arr[4] = r8;
arr[5] = r9;
arr[6] = r10;
arr[7] = r11;
arr[8] = r12;
arr[9] = r13;
}
#else
// If we don't have architecture-specific optimized support,
// just fall back to setjmp-based implementation.
#undef MICROPY_GCREGS_SETJMP
#define MICROPY_GCREGS_SETJMP (1)
#endif // Arch-specific selection
#endif // !MICROPY_GCREGS_SETJMP
// If MICROPY_GCREGS_SETJMP was requested explicitly, or if
// we enabled it as a fallback above.
#if MICROPY_GCREGS_SETJMP
#include <setjmp.h>
typedef jmp_buf regs_t;
STATIC void gc_helper_get_regs(regs_t arr) {
setjmp(arr);
}
#endif // MICROPY_GCREGS_SETJMP
void gc_collect(void) {
gc_collect_start();
regs_t regs;
gc_helper_get_regs(regs);
// GC stack (and regs because we captured them)
void **regs_ptr = (void**)(void*)&regs;
gc_collect_root(regs_ptr, ((mp_uint_t)MP_STATE_VM(stack_top) - (mp_uint_t)&regs) / sizeof(mp_uint_t));
#if MICROPY_EMIT_NATIVE
mp_unix_mark_exec();
#endif
gc_collect_end();
}
#endif //MICROPY_ENABLE_GC

272
mpy-cross/main.c Normal file
View File

@ -0,0 +1,272 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013-2016 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include "py/mpstate.h"
#include "py/compile.h"
#include "py/runtime.h"
#include "py/gc.h"
#include "py/stackctrl.h"
// Command line options, with their defaults
STATIC uint emit_opt = MP_EMIT_OPT_NONE;
mp_uint_t mp_verbose_flag = 0;
// Heap size of GC heap (if enabled)
// Make it larger on a 64 bit machine, because pointers are larger.
long heap_size = 1024*1024 * (sizeof(mp_uint_t) / 4);
STATIC void stderr_print_strn(void *env, const char *str, mp_uint_t len) {
(void)env;
ssize_t dummy = write(STDERR_FILENO, str, len);
(void)dummy;
}
STATIC const mp_print_t mp_stderr_print = {NULL, stderr_print_strn};
STATIC int compile_and_save(const char *file, const char *output_file) {
mp_lexer_t *lex = mp_lexer_new_from_file(file);
if (lex == NULL) {
printf("MemoryError: lexer could not allocate memory\n");
return 1;
}
nlr_buf_t nlr;
if (nlr_push(&nlr) == 0) {
qstr source_name = lex->source_name;
#if MICROPY_PY___FILE__
if (input_kind == MP_PARSE_FILE_INPUT) {
mp_store_global(MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name));
}
#endif
mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT);
mp_raw_code_t *rc = mp_compile_to_raw_code(&parse_tree, source_name, emit_opt, false);
vstr_t vstr;
vstr_init(&vstr, 16);
if (output_file == NULL) {
vstr_add_str(&vstr, file);
vstr_cut_tail_bytes(&vstr, 2);
vstr_add_str(&vstr, "mpy");
} else {
vstr_add_str(&vstr, output_file);
}
mp_raw_code_save_file(rc, vstr_null_terminated_str(&vstr));
vstr_clear(&vstr);
nlr_pop();
return 0;
} else {
// uncaught exception
mp_obj_print_exception(&mp_stderr_print, (mp_obj_t)nlr.ret_val);
return 1;
}
}
STATIC int usage(char **argv) {
printf(
"usage: %s [<opts>] [-X <implopt>] <input filename>\n"
"Options:\n"
"-o : output file for compiled bytecode\n"
"-v : verbose (trace various operations); can be multiple\n"
"-O[N] : apply bytecode optimizations of level N\n"
"\n"
"Target specific options:\n"
"-msmall-int-bits=number : set the maximum bits used to encode a small-int\n"
"-mno-unicode : don't support unicode in compiled strings\n"
"-mcache-lookup-bc : cache map lookups in the bytecode\n"
"\n"
"Implementation specific options:\n", argv[0]
);
int impl_opts_cnt = 0;
printf(
" emit={bytecode,native,viper} -- set the default code emitter\n"
);
impl_opts_cnt++;
printf(
" heapsize=<n> -- set the heap size for the GC (default %ld)\n"
, heap_size);
impl_opts_cnt++;
if (impl_opts_cnt == 0) {
printf(" (none)\n");
}
return 1;
}
// Process options which set interpreter init options
STATIC void pre_process_options(int argc, char **argv) {
for (int a = 1; a < argc; a++) {
if (argv[a][0] == '-') {
if (strcmp(argv[a], "-X") == 0) {
if (a + 1 >= argc) {
exit(usage(argv));
}
if (strcmp(argv[a + 1], "emit=bytecode") == 0) {
emit_opt = MP_EMIT_OPT_BYTECODE;
} else if (strcmp(argv[a + 1], "emit=native") == 0) {
emit_opt = MP_EMIT_OPT_NATIVE_PYTHON;
} else if (strcmp(argv[a + 1], "emit=viper") == 0) {
emit_opt = MP_EMIT_OPT_VIPER;
} else if (strncmp(argv[a + 1], "heapsize=", sizeof("heapsize=") - 1) == 0) {
char *end;
heap_size = strtol(argv[a + 1] + sizeof("heapsize=") - 1, &end, 0);
// Don't bring unneeded libc dependencies like tolower()
// If there's 'w' immediately after number, adjust it for
// target word size. Note that it should be *before* size
// suffix like K or M, to avoid confusion with kilowords,
// etc. the size is still in bytes, just can be adjusted
// for word size (taking 32bit as baseline).
bool word_adjust = false;
if ((*end | 0x20) == 'w') {
word_adjust = true;
end++;
}
if ((*end | 0x20) == 'k') {
heap_size *= 1024;
} else if ((*end | 0x20) == 'm') {
heap_size *= 1024 * 1024;
}
if (word_adjust) {
heap_size = heap_size * BYTES_PER_WORD / 4;
}
} else {
exit(usage(argv));
}
a++;
}
}
}
}
MP_NOINLINE int main_(int argc, char **argv) {
mp_stack_set_limit(40000 * (BYTES_PER_WORD / 4));
pre_process_options(argc, argv);
char *heap = malloc(heap_size);
gc_init(heap, heap + heap_size);
mp_init();
mp_obj_list_init(mp_sys_path, 0);
mp_obj_list_init(mp_sys_argv, 0);
// set default compiler configuration
mp_dynamic_compiler.small_int_bits = 31;
mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 0;
mp_dynamic_compiler.py_builtins_str_unicode = 1;
const char *input_file = NULL;
const char *output_file = NULL;
// parse main options
for (int a = 1; a < argc; a++) {
if (argv[a][0] == '-') {
if (strcmp(argv[a], "-X") == 0) {
a += 1;
} else if (strcmp(argv[a], "-v") == 0) {
mp_verbose_flag++;
} else if (strncmp(argv[a], "-O", 2) == 0) {
if (unichar_isdigit(argv[a][2])) {
MP_STATE_VM(mp_optimise_value) = argv[a][2] & 0xf;
} else {
MP_STATE_VM(mp_optimise_value) = 0;
for (char *p = argv[a] + 1; *p && *p == 'O'; p++, MP_STATE_VM(mp_optimise_value)++);
}
} else if (strcmp(argv[a], "-o") == 0) {
if (a + 1 >= argc) {
exit(usage(argv));
}
a += 1;
output_file = argv[a];
} else if (strncmp(argv[a], "-msmall-int-bits=", sizeof("-msmall-int-bits=") - 1) == 0) {
char *end;
mp_dynamic_compiler.small_int_bits =
strtol(argv[a] + sizeof("-msmall-int-bits=") - 1, &end, 0);
if (*end) {
return usage(argv);
}
// TODO check that small_int_bits is within range of host's capabilities
} else if (strcmp(argv[a], "-mno-cache-lookup-bc") == 0) {
mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 0;
} else if (strcmp(argv[a], "-mcache-lookup-bc") == 0) {
mp_dynamic_compiler.opt_cache_map_lookup_in_bytecode = 1;
} else if (strcmp(argv[a], "-mno-unicode") == 0) {
mp_dynamic_compiler.py_builtins_str_unicode = 0;
} else if (strcmp(argv[a], "-municode") == 0) {
mp_dynamic_compiler.py_builtins_str_unicode = 1;
} else {
return usage(argv);
}
} else {
if (input_file != NULL) {
mp_printf(&mp_stderr_print, "multiple input files\n");
exit(1);
}
input_file = argv[a];
}
}
if (input_file == NULL) {
mp_printf(&mp_stderr_print, "no input file\n");
exit(1);
}
int ret = compile_and_save(input_file, output_file);
#if MICROPY_PY_MICROPYTHON_MEM_INFO
if (mp_verbose_flag) {
mp_micropython_mem_info(0, NULL);
}
#endif
mp_deinit();
return ret & 0xff;
}
int main(int argc, char **argv) {
mp_stack_ctrl_init();
return main_(argc, argv);
}
uint mp_import_stat(const char *path) {
(void)path;
return MP_IMPORT_STAT_NO_EXIST;
}
void nlr_jump_fail(void *val) {
printf("FATAL: uncaught NLR %p\n", val);
exit(1);
}

122
mpy-cross/mpconfigport.h Normal file
View File

@ -0,0 +1,122 @@
/*
* This file is part of the MicroPython project, http://micropython.org/
*
* The MIT License (MIT)
*
* Copyright (c) 2013-2015 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
// options to control how MicroPython is built
#define MICROPY_ALLOC_PATH_MAX (PATH_MAX)
#define MICROPY_PERSISTENT_CODE_LOAD (0)
#define MICROPY_PERSISTENT_CODE_SAVE (1)
#define MICROPY_EMIT_X64 (0)
#define MICROPY_EMIT_X86 (0)
#define MICROPY_EMIT_THUMB (0)
#define MICROPY_EMIT_INLINE_THUMB (0)
#define MICROPY_EMIT_INLINE_THUMB_ARMV7M (0)
#define MICROPY_EMIT_INLINE_THUMB_FLOAT (0)
#define MICROPY_EMIT_ARM (0)
#define MICROPY_DYNAMIC_COMPILER (1)
#define MICROPY_COMP_CONST_FOLDING (1)
#define MICROPY_COMP_MODULE_CONST (1)
#define MICROPY_COMP_CONST (1)
#define MICROPY_COMP_DOUBLE_TUPLE_ASSIGN (1)
#define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (1)
#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (0)
#define MICROPY_ENABLE_RUNTIME (0)
#define MICROPY_ENABLE_GC (1)
#define MICROPY_STACK_CHECK (1)
#define MICROPY_HELPER_LEXER_UNIX (1)
#define MICROPY_LONGINT_IMPL (MICROPY_LONGINT_IMPL_MPZ)
#define MICROPY_ENABLE_SOURCE_LINE (1)
#define MICROPY_ENABLE_DOC_STRING (0)
#define MICROPY_ERROR_REPORTING (MICROPY_ERROR_REPORTING_DETAILED)
#define MICROPY_WARNINGS (1)
#define MICROPY_FLOAT_IMPL (MICROPY_FLOAT_IMPL_DOUBLE)
#define MICROPY_CPYTHON_COMPAT (1)
#define MICROPY_PY_BUILTINS_STR_UNICODE (1)
// Define to 1 to use undertested inefficient GC helper implementation
// (if more efficient arch-specific one is not available).
#ifndef MICROPY_GCREGS_SETJMP
#ifdef __mips__
#define MICROPY_GCREGS_SETJMP (1)
#else
#define MICROPY_GCREGS_SETJMP (0)
#endif
#endif
#define MICROPY_PY___FILE__ (0)
#define MICROPY_PY_ARRAY (0)
#define MICROPY_PY_ATTRTUPLE (0)
#define MICROPY_PY_COLLECTIONS (0)
#define MICROPY_PY_MATH (0)
#define MICROPY_PY_CMATH (0)
#define MICROPY_PY_GC (0)
#define MICROPY_PY_IO (0)
#define MICROPY_PY_SYS (0)
// type definitions for the specific machine
#ifdef __LP64__
typedef long mp_int_t; // must be pointer size
typedef unsigned long mp_uint_t; // must be pointer size
#else
// These are definitions for machines where sizeof(int) == sizeof(void*),
// regardless for actual size.
typedef int mp_int_t; // must be pointer size
typedef unsigned int mp_uint_t; // must be pointer size
#endif
#define BYTES_PER_WORD sizeof(mp_int_t)
// Cannot include <sys/types.h>, as it may lead to symbol name clashes
#if _FILE_OFFSET_BITS == 64 && !defined(__LP64__)
typedef long long mp_off_t;
#else
typedef long mp_off_t;
#endif
typedef void *machine_ptr_t; // must be of pointer size
typedef const void *machine_const_ptr_t; // must be of pointer size
#define MP_PLAT_PRINT_STRN(str, len) (void)0
#ifndef MP_NOINLINE
#define MP_NOINLINE __attribute__((noinline))
#endif
// We need to provide a declaration/definition of alloca()
#ifdef __FreeBSD__
#include <stdlib.h>
#else
#include <alloca.h>
#endif
#include <stdint.h>

1
mpy-cross/mphalport.h Normal file
View File

@ -0,0 +1 @@
// empty file

1
mpy-cross/qstrdefsport.h Normal file
View File

@ -0,0 +1 @@
// qstrs specific to this port