From 89ff506513d52c0c415b2cf45335d60cefac527d Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Wed, 12 Dec 2018 16:50:55 +1100 Subject: [PATCH] py: Update and rework build system for including external C modules. How to use this feature is documented in docs/develop/cmodules.rst. --- docs/develop/cmodules.rst | 163 ++++++++++++++++++++++++++++++++++++ docs/develop/index.rst | 15 ++++ docs/index.rst | 1 + docs/reference/cmodules.rst | 86 ------------------- docs/reference/index.rst | 8 -- mpy-cross/Makefile | 2 + ports/stm32/Makefile | 2 +- py/mkenv.mk | 1 - py/mkrules.mk | 17 ++-- py/objmodule.c | 2 + py/py.mk | 19 ++++- tools/gen-cmodules.py | 28 ------- 12 files changed, 205 insertions(+), 139 deletions(-) create mode 100644 docs/develop/cmodules.rst create mode 100644 docs/develop/index.rst delete mode 100644 docs/reference/cmodules.rst delete mode 100755 tools/gen-cmodules.py diff --git a/docs/develop/cmodules.rst b/docs/develop/cmodules.rst new file mode 100644 index 0000000000..c3de90a0f6 --- /dev/null +++ b/docs/develop/cmodules.rst @@ -0,0 +1,163 @@ +MicroPython external C modules +============================== + +When developing modules for use with MicroPython you may find you run into +limitations with the Python environment, often due to an inability to access +certain hardware resources or Python speed limitations. + +If your limitations can't be resolved with suggestions in :ref:`speed_python`, +writing some or all of your module in C is a viable option. + +If your module is designed to access or work with commonly available +hardware or libraries please consider implementing it inside the MicroPython +source tree alongside similar modules and submitting it as a pull request. +If however you're targeting obscure or proprietary systems it may make +more sense to keep this external to the main MicroPython repository. + +This chapter describes how to compile such external modules into the +MicroPython executable or firmware image. + + +Structure of an external C module +--------------------------------- + +A MicroPython user C module is a directory with the following files: + +* ``*.c`` and/or ``*.h`` source code files for your module. + + These will typically include the low level functionality being implemented and + the MicroPython binding functions to expose the functions and module(s). + + Currently the best reference for writing these functions/modules is + to find similar modules within the MicroPython tree and use them as examples. + +* ``micropython.mk`` contains the Makefile fragment for this module. + + ``$(USERMOD_DIR)`` is available in ``micropython.mk`` as the path to your + module directory. As it's redefined for each c module, is should be expanded + in your ``micropython.mk`` to a local make variable, + eg ``EXAMPLE_MOD_DIR := $(USERMOD_DIR)`` + + Your ``micropython.mk`` must add your modules C files relative to your + expanded copy of ``$(USERMOD_DIR)`` to ``SRC_USERMOD``, eg + ``SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c`` + + If you have custom ``CFLAGS`` settings or include folders to define, these + should be added to ``CFLAGS_USERMOD``. + + See below for full usage example. + + +Basic Example +------------- + +This simple module named ``example`` provides a single function +``example.add_ints(a, b)`` which adds the two integer args together and returns +the result. + +Directory:: + + example/ + ├── example.c + └── micropython.mk + + +``example.c`` + +.. code-block:: c + + // Include required definitions first. + #include "py/obj.h" + #include "py/runtime.h" + #include "py/builtin.h" + + #define MODULE_EXAMPLE_ENABLED (1) + + // This is the function which will be called from Python as example.add_ints(a, b). + STATIC mp_obj_t example_add_ints(mp_obj_t a_obj, mp_obj_tab_obj) { + // Extract the ints from the micropython input objects + int a = mp_obj_get_int(a_obj); + int b = mp_obj_get_int(b_obj); + + // Calculate the addition and convert to MicroPython object. + return mp_obj_new_int(a + b); + } + // Define a Python reference to the function above + STATIC MP_DEFINE_CONST_FUN_OBJ_1(example_add_ints_obj, example_add_ints); + + // Define all properties of the example module. + // Table entries are key/value pairs of the attribute name (a string) + // and the MicroPython object reference. + // All identifiers and strings are written as MP_QSTR_xxx and will be + // optimized to word-sized integers by the build system (interned strings). + STATIC const mp_rom_map_elem_t example_module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example) }, + { MP_ROM_QSTR(MP_QSTR_add_ints), MP_ROM_PTR(&example_add_ints_obj) }, + }; + STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table); + + // Define module object. + const mp_obj_module_t example_user_cmodule = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t*)&example_module_globals, + }; + + // Register the module to make it available in Python + MP_REGISTER_MODULE(MP_QSTR_example, example_user_cmodule, MODULE_EXAMPLE_ENABLED); + + +``micropython.mk`` + +.. code-block:: make + + EXAMPLE_MOD_DIR := $(USERMOD_DIR) + + # Add all C files to SRC_USERMOD. + SRC_USERMOD += $(EXAMPLE_MOD_DIR)/example.c + + # We can add our module folder to include paths if needed + # This is not actually needed in this example. + CFLAGS_USERMOD += -I$(EXAMPLE_MOD_DIR) + + +Compiling the cmodule into MicroPython +-------------------------------------- + +To build such a module, compile MicroPython (see `getting started +`_) with an +extra ``make`` flag named ``USER_C_MODULES`` set to the directory containing +all modules you want included (not to the module itself). For example: + + +Directory:: + + my_project/ + ├── modules/ + │ └──example/ + │ ├──example.c + │ └──micropython.mk + └── micropython/ + ├──ports/ + ... ├──stm32/ + ... + +Building for stm32 port: + +.. code-block:: bash + + cd my_project/micropython/ports/stm32 + make USER_C_MODULES=../../../modules all + + +Module usage in MicroPython +--------------------------- + +Once built into your copy of MicroPython, the module implemented +in ``example.c`` above can now be accessed in Python just +like any other builtin module, eg + +.. code-block:: python + + import example + print(example.add_ints(1, 3)) + # should display 4 diff --git a/docs/develop/index.rst b/docs/develop/index.rst new file mode 100644 index 0000000000..6b7b3c3910 --- /dev/null +++ b/docs/develop/index.rst @@ -0,0 +1,15 @@ +Developing and building MicroPython +=================================== + +This chapter describes modules (function and class libraries) which are built +into MicroPython. There are a few categories of such modules: + +This chapter describes some options for extending MicroPython in C. Note +that it doesn't aim to be a complete guide for developing with MicroPython. +See the `getting started guide +`_ for further information. + +.. toctree:: + :maxdepth: 1 + + cmodules.rst diff --git a/docs/index.rst b/docs/index.rst index 235185b6c2..c0417c227c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,6 +6,7 @@ MicroPython documentation and references library/index.rst reference/index.rst genrst/index.rst + develop/index.rst license.rst pyboard/quickref.rst esp8266/quickref.rst diff --git a/docs/reference/cmodules.rst b/docs/reference/cmodules.rst deleted file mode 100644 index f361af4a38..0000000000 --- a/docs/reference/cmodules.rst +++ /dev/null @@ -1,86 +0,0 @@ -Extending MicroPython with C -============================ - -Some specialized code would be unacceptably slow or needs to access hardware in -a way that cannot be done from MicroPython. Therefore, it supports a way of -extending the language with custom modules written in C. But before you consider -writing a module in C, please take a look at :ref:`speed_python`. - -`Unlike CPython `_, these -modules are (currently) embedded directly in the program image instead of being -dynamically loaded. This requires a `custom build of MicroPython -`_. - - -Writing a module ----------------- - -A module is a directory with the following files: - - * ``micropython.mk``, which contains the Makefile fragment for this module. - * All C files you would like included. - -Put the required build commands in ``micropython.mk``. For a simple module, you -will only have to add the file paths to ``SRC_MOD``, which will include these C -files in the build: - -.. highlight:: make -.. code:: - - # Add all C files to SRC_MOD. - SRC_MOD += $(USER_C_MODULES)/example/example.c - -This is a very bare bones module named ``example`` that provides -``example.double(x)``. Note that the name of the module must be equal to the -directory name and is also used in the name of the ``mp_obj_module_t`` object at -the bottom. - -.. highlight:: c -.. code:: - - // Include required definitions first. - #include "py/obj.h" - #include "py/runtime.h" - - // This is the function you will call using example.double(n). - STATIC mp_obj_t example_double(mp_obj_t x_obj) { - // Check input value and convert it to a C type. - if (!MP_OBJ_IS_SMALL_INT(x_obj)) { - mp_raise_ValueError("x is not a small int"); - } - int x = mp_obj_int_get_truncated(x_obj); - - // Calculate the double, and convert back to MicroPython object. - return mp_obj_new_int(x + x); - } - STATIC MP_DEFINE_CONST_FUN_OBJ_1(example_double_obj, example_double); - - // Define all properties of the example module, which currently are the name (a - // string) and a function. - // All identifiers and strings are written as MP_QSTR_xxx and will be - // optimized to word-sized integers by the build system (interned strings). - STATIC const mp_rom_map_elem_t example_module_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_example) }, - { MP_ROM_QSTR(MP_QSTR_double), MP_ROM_PTR(&example_double_obj) }, - }; - STATIC MP_DEFINE_CONST_DICT(example_module_globals, example_module_globals_table); - - // Define module object. - const mp_obj_module_t example_user_cmodule = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t*)&example_module_globals, - }; - - -Using a module --------------- - -To build such a module, compile MicroPython (see `getting started -`_) with an -extra ``make`` flag named ``USER_C_MODULES`` set to the directory containing -all modules you want included (not to the module itself!). For example: - -.. highlight:: shell -.. code:: - - $ make USER_C_MODULES=path-to-modules-folder all diff --git a/docs/reference/index.rst b/docs/reference/index.rst index e2e08a7f72..d0c7f69de9 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -26,11 +26,3 @@ implementation and the best practices to use them. constrained.rst packages.rst asm_thumb2_index.rst - cmodules.rst - -.. only:: port_pyboard - - .. toctree:: - :maxdepth: 1 - - asm_thumb2_index.rst diff --git a/mpy-cross/Makefile b/mpy-cross/Makefile index c42c2c5abb..4ff96fc80c 100644 --- a/mpy-cross/Makefile +++ b/mpy-cross/Makefile @@ -9,6 +9,8 @@ override undefine MICROPY_FORCE_32BIT override undefine CROSS_COMPILE override undefine FROZEN_DIR override undefine FROZEN_MPY_DIR +override undefine USER_C_MODULES +override undefine SRC_MOD override undefine BUILD override undefine PROG endif diff --git a/ports/stm32/Makefile b/ports/stm32/Makefile index 4fddc6c7d3..b4c7a15c13 100644 --- a/ports/stm32/Makefile +++ b/ports/stm32/Makefile @@ -494,7 +494,7 @@ $(BUILD)/firmware.hex: $(BUILD)/firmware.elf $(BUILD)/firmware.elf: $(OBJ) $(ECHO) "LINK $@" - $(Q)$(LD) $(LDFLAGS) -o $@ $^ $(LIBS) + $(Q)$(LD) $(LDFLAGS) -o $@ $^ $(LDFLAGS_MOD) $(LIBS) $(Q)$(SIZE) $@ PLLVALUES = boards/pllvalues.py diff --git a/py/mkenv.mk b/py/mkenv.mk index 5f4b11b6be..87e92ec6f9 100644 --- a/py/mkenv.mk +++ b/py/mkenv.mk @@ -61,7 +61,6 @@ endif MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py MPY_CROSS = $(TOP)/mpy-cross/mpy-cross MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py -GEN_CMODULES = $(PYTHON) $(TOP)/tools/gen-cmodules.py all: .PHONY: all diff --git a/py/mkrules.mk b/py/mkrules.mk index 65d86834ef..caa9527c70 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -20,12 +20,12 @@ endif # can be located. By following this scheme, it allows a single build rule # to be used to compile all .c files. -vpath %.S . $(TOP) +vpath %.S . $(TOP) $(USER_C_MODULES) $(BUILD)/%.o: %.S $(ECHO) "CC $<" $(Q)$(CC) $(CFLAGS) -c -o $@ $< -vpath %.s . $(TOP) +vpath %.s . $(TOP) $(USER_C_MODULES) $(BUILD)/%.o: %.s $(ECHO) "AS $<" $(Q)$(AS) -o $@ $< @@ -42,14 +42,14 @@ $(Q)$(CC) $(CFLAGS) -c -MD -o $@ $< $(RM) -f $(@:.o=.d) endef -vpath %.c . $(TOP) +vpath %.c . $(TOP) $(USER_C_MODULES) $(BUILD)/%.o: %.c $(call compile_c) QSTR_GEN_EXTRA_CFLAGS += -DNO_QSTR QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp -vpath %.c . $(TOP) +vpath %.c . $(TOP) $(USER_C_MODULES) $(BUILD)/%.pp: %.c $(ECHO) "PreProcess $<" @@ -105,7 +105,7 @@ endif ifneq ($(FROZEN_MPY_DIR),) # to build the MicroPython cross compiler $(TOP)/mpy-cross/mpy-cross: $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/ports/windows/fmode.c - $(Q)$(MAKE) -C $(TOP)/mpy-cross USER_C_MODULES= + $(Q)$(MAKE) -C $(TOP)/mpy-cross # make a list of all the .py files that need compiling and freezing FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==') @@ -123,13 +123,6 @@ $(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generate $(Q)$(MPY_TOOL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@ endif -# to build a list of modules for py/objmodule.c. -ifneq ($(USER_C_MODULES),) -$(BUILD)/genhdr/cmodules.h: | $(HEADER_BUILD)/mpversion.h - @$(ECHO) "GEN $@" - $(Q)$(GEN_CMODULES) $(USER_C_MODULES) > $@ -endif - ifneq ($(PROG),) # Build a standalone executable (unix does this) diff --git a/py/objmodule.c b/py/objmodule.c index 04d210260d..9191c73ec3 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -31,6 +31,8 @@ #include "py/runtime.h" #include "py/builtin.h" +#include "genhdr/moduledefs.h" + STATIC void module_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { (void)kind; mp_obj_module_t *self = MP_OBJ_TO_PTR(self_in); diff --git a/py/py.mk b/py/py.mk index ad7d122048..0fbc9f14bb 100644 --- a/py/py.mk +++ b/py/py.mk @@ -131,9 +131,20 @@ endif # External modules written in C. ifneq ($(USER_C_MODULES),) -CFLAGS_MOD += -DMICROPY_CMODULES_INCLUDE_H='"genhdr/cmodules.h"' -include $(USER_C_MODULES)/*/micropython.mk -SRC_QSTR += $(BUILD)/genhdr/cmodules.h +# pre-define USERMOD variables as expanded so that variables are immediate +# expanded as they're added to them +SRC_USERMOD := +CFLAGS_USERMOD := +LDFLAGS_USERMOD := +$(foreach module, $(wildcard $(USER_C_MODULES)/*/micropython.mk), \ + $(eval USERMOD_DIR = $(patsubst %/,%,$(dir $(module))))\ + $(info Including User C Module from $(USERMOD_DIR))\ + $(eval include $(module))\ +) + +SRC_MOD += $(patsubst $(USER_C_MODULES)/%.c,%.c,$(SRC_USERMOD)) +CFLAGS_MOD += $(CFLAGS_USERMOD) +LDFLAGS_MOD += $(LDFLAGS_USERMOD) endif # py object files @@ -335,6 +346,8 @@ $(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER @$(ECHO) "GEN $@" $(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@ +SRC_QSTR += $(HEADER_BUILD)/moduledefs.h + # Force nlr code to always be compiled with space-saving optimisation so # that the function preludes are of a minimal and predictable form. $(PY_BUILD)/nlr%.o: CFLAGS += -Os diff --git a/tools/gen-cmodules.py b/tools/gen-cmodules.py deleted file mode 100755 index 524e3c03d3..0000000000 --- a/tools/gen-cmodules.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python - -# Generate genhdr/cmodules.h for inclusion in py/objmodule.c. - -from __future__ import print_function - -import sys -import os -from glob import glob - -def update_modules(path): - modules = [] - for module in sorted(os.listdir(path)): - if not os.path.isfile('%s/%s/micropython.mk' % (path, module)): - continue # not a module - modules.append(module) - - # Print header file for all external modules. - print('// Automatically generated by genmodules.py.\n') - for module in modules: - print('extern const struct _mp_obj_module_t %s_user_cmodule;' % module) - print('\n#define MICROPY_EXTRA_BUILTIN_MODULES \\') - for module in modules: - print(' { MP_ROM_QSTR(MP_QSTR_%s), MP_ROM_PTR(&%s_user_cmodule) }, \\' % (module, module)) - print() - -if __name__ == '__main__': - update_modules(sys.argv[1])