djsundog/circuitpython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

py/make_root_pointers: Add MP_REGISTER_ROOT_POINTER parser/generator.

Browse Source 
This adds new compile-time infrastructure to parse source code files for
`MP_REGISTER_ROOT_POINTER()` and generates a new `root_pointers.h` header
file containing the collected declarations.  This works the same as the
existing `MP_REGISTER_MODULE()` feature.

Signed-off-by: David Lechner <david@pybricks.com>
This commit is contained in:
David Lechner 2022-07-01 12:29:08 -05:00 committed by Damien George
parent a8d78cc398
commit fc3d7ae11b
10 changed files with 154 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ports/cc3200/bootmgr/bootloader.mk
View File

@ -134,3 +134,7 @@ $(HEADER_BUILD)/mpversion.h: | $(HEADER_BUILD)
# Create an empty "moduledefs.h" needed by py/mkrules.mk
$(HEADER_BUILD)/moduledefs.h: | $(HEADER_BUILD)
touch $@
# Create an empty "root_pointers.h" needed by py/mkrules.mk
$(HEADER_BUILD)/root_pointers.h: | $(HEADER_BUILD)
touch $@

6
ports/stm32/mboot/Makefile
View File

@ -213,6 +213,7 @@ PREFIX_FILE = ../boards/stm32f4xx_prefix.c
BOARD_PINS = $(BOARD_DIR)/pins.csv
HEADER_BUILD = $(BUILD)/genhdr
GEN_QSTRDEFS_GENERATED = $(HEADER_BUILD)/qstrdefs.generated.h
GEN_ROOT_POINTERS = $(HEADER_BUILD)/root_pointers.h
GEN_PINS_SRC = $(BUILD)/pins_$(BOARD).c
GEN_PINS_HDR = $(HEADER_BUILD)/pins.h
GEN_PINS_QSTR = $(HEADER_BUILD)/pins_qstr.h
@ -220,7 +221,7 @@ GEN_PINS_AF_CONST = $(HEADER_BUILD)/pins_af_const.h
GEN_PINS_AF_DEFS = $(HEADER_BUILD)/pins_af_defs.h
GEN_PINS_AF_PY = $(BUILD)/pins_af.py
$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_PINS_AF_DEFS)
$(OBJ): $(GEN_QSTRDEFS_GENERATED) $(GEN_ROOT_POINTERS) $(GEN_PINS_AF_DEFS)
$(HEADER_BUILD):
$(MKDIR) -p $(BUILD)/genhdr
@ -228,6 +229,9 @@ $(HEADER_BUILD):
$(GEN_QSTRDEFS_GENERATED): | $(HEADER_BUILD)
$(Q)echo "// empty" > $@
$(GEN_ROOT_POINTERS): | $(HEADER_BUILD)
$(Q)echo "// empty" > $@
$(GEN_PINS_AF_DEFS): $(BOARD_PINS) $(MAKE_PINS) ../$(AF_FILE) $(PREFIX_FILE) | $(HEADER_BUILD)
$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(MAKE_PINS) --board $(BOARD_PINS) --af ../$(AF_FILE) \

21
ports/windows/msvc/genhdr.targets
View File

@ -4,7 +4,7 @@
<Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>
<!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
<Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeRootPointers;MakeQstrData">
</Target>
<PropertyGroup>
@ -15,6 +15,7 @@
<QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
<QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
<ModuleDefsCollected>$(DestDir)/moduledefs.collected</ModuleDefsCollected>
<RootPointersCollected>$(DestDir)/root_pointers.collected</RootPointersCollected>
<PyPython Condition="'$(PyPython)' == ''">$(MICROPY_CPYTHON3)</PyPython>
<PyPython Condition="'$(PyPython)' == ''">python</PyPython>
<CLToolExe Condition="'$(CLToolExe)' == ''">cl.exe</CLToolExe>
@ -45,7 +46,7 @@
using(var outFile = System.IO.File.CreateText(OutputFile)) {
foreach(var inFile in InputFiles)
foreach(var line in System.IO.File.ReadAllLines(inFile))
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR") || line.Contains("MP_REGISTER"))
outFile.WriteLine( line );
}
]]>
@ -114,6 +115,20 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
</Target>
<Target Name="CollectRootPointers" DependsOnTargets="MakeQstrDefs" Inputs="$(DestDir)qstr.i.last" Outputs="$(RootPointersCollected)">
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split root_pointer $(DestDir)qstr.i.last $(DestDir)root_pointer _"/>
<Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat root_pointer _ $(DestDir)root_pointer $(RootPointersCollected)"/>
</Target>
<Target Name="MakeRootPointers" DependsOnTargets="CollectRootPointers" Inputs="$(PySrcDir)make_root_pointers.py;$(RootPointersCollected)" Outputs="$(DestDir)root_pointers.h">
<PropertyGroup>
<DestFile>$(DestDir)root_pointers.h</DestFile>
<TmpFile>$(DestFile).tmp</TmpFile>
</PropertyGroup>
<Exec Command="$(PyPython) $(PySrcDir)make_root_pointers.py $(RootPointersCollected) > $(TmpFile)"/>
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
</Target>
<Target Name="MakeQstrData" DependsOnTargets="MakeQstrDefs" Inputs="$(QstrDefsCollected);$(PyQstrDefs);$(QstrDefs)" Outputs="$(QstrGen)">
<PropertyGroup>
<TmpFile>$(QstrGen).tmp</TmpFile>
@ -132,7 +147,7 @@ using(var outFile = System.IO.File.CreateText(OutputFile)) {
<MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
</Target>
<Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
<Target Name="FreezeModules" Condition="'$(FrozenManifest)' != ''" DependsOnTargets="MakeQstrData;MakeRootPointers" Inputs="$(FrozenManifest)" Outputs="$(PyBuildDir)frozen_content.c">
<ItemGroup>
<ClCompile Include="$(PyBuildDir)frozen_content.c"/>
<ClCompile>

55
py/make_root_pointers.py Normal file
View File

@ -0,0 +1,55 @@
"""
This pre-processor parses a single file containing a list of
MP_REGISTER_ROOT_POINTER(variable declaration) items.
These are used to generate a header with the required entries for
"struct _mp_state_vm_t" in py/mpstate.h
"""
from __future__ import print_function
import argparse
import io
import re
PATTERN = re.compile(r"MP_REGISTER_ROOT_POINTER\((.*?)\);")
def find_root_pointer_registrations(filename):
"""Find any MP_REGISTER_ROOT_POINTER definitions in the provided file.
:param str filename: path to file to check
:return: List[variable_declaration]
"""
with io.open(filename, encoding="utf-8") as c_file_obj:
return set(re.findall(PATTERN, c_file_obj.read()))
def generate_root_pointer_header(root_pointers):
"""Generate header with root pointer entries.
:param List[variable_declaration] root_pointers: root pointer declarations
:return: None
"""
# Print header file for all external modules.
print("// Automatically generated by make_root_pointers.py.")
print()
for item in root_pointers:
print(item, end=";")
print()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("file", nargs=1, help="file with MP_REGISTER_ROOT_POINTER definitions")
args = parser.parse_args()
root_pointers = find_root_pointer_registrations(args.file[0])
generate_root_pointer_header(sorted(root_pointers))
if __name__ == "__main__":
main()

11
py/makeqstrdefs.py
View File

@ -24,6 +24,9 @@ _MODE_COMPRESS = "compress"
# Extract MP_REGISTER_MODULE(...) macros.
_MODE_MODULE = "module"
# Extract MP_REGISTER_ROOT_POINTER(...) macros.
_MODE_ROOT_POINTER = "root_pointer"
def is_c_source(fname):
return os.path.splitext(fname)[1] in [".c"]
@ -90,6 +93,8 @@ def process_file(f):
re_match = re.compile(r'MP_COMPRESSED_ROM_TEXT\("([^"]*)"\)')
elif args.mode == _MODE_MODULE:
re_match = re.compile(r"MP_REGISTER_MODULE\(.*?,\s*.*?\);")
elif args.mode == _MODE_ROOT_POINTER:
re_match = re.compile(r"MP_REGISTER_ROOT_POINTER\(.*?\);")
output = []
last_fname = None
for line in f:
@ -111,7 +116,7 @@ def process_file(f):
if args.mode == _MODE_QSTR:
name = match.replace("MP_QSTR_", "")
output.append("Q(" + name + ")")
elif args.mode in (_MODE_COMPRESS, _MODE_MODULE):
elif args.mode in (_MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
output.append(match)
if last_fname:
@ -148,6 +153,8 @@ def cat_together():
mode_full = "Compressed data"
elif args.mode == _MODE_MODULE:
mode_full = "Module registrations"
elif args.mode == _MODE_ROOT_POINTER:
mode_full = "Root pointer registrations"
if old_hash != new_hash:
print(mode_full, "updated")
try:
@ -208,7 +215,7 @@ if __name__ == "__main__":
args.output_dir = sys.argv[4]
args.output_file = None if len(sys.argv) == 5 else sys.argv[5] # Unused for command=split
if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE):
if args.mode not in (_MODE_QSTR, _MODE_COMPRESS, _MODE_MODULE, _MODE_ROOT_POINTER):
print("error: mode %s unrecognised" % sys.argv[2])
sys.exit(2)

30
py/mkrules.cmake
View File

@ -11,6 +11,9 @@ set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h")
set(MICROPY_MODULEDEFS_SPLIT "${MICROPY_GENHDR_DIR}/moduledefs.split")
set(MICROPY_MODULEDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/moduledefs.collected")
set(MICROPY_MODULEDEFS "${MICROPY_GENHDR_DIR}/moduledefs.h")
set(MICROPY_ROOT_POINTERS_SPLIT "${MICROPY_GENHDR_DIR}/root_pointers.split")
set(MICROPY_ROOT_POINTERS_COLLECTED "${MICROPY_GENHDR_DIR}/root_pointers.collected")
set(MICROPY_ROOT_POINTERS "${MICROPY_GENHDR_DIR}/root_pointers.h")
# Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen
# manifest handling is at the end of this file.
@ -46,6 +49,7 @@ target_sources(${MICROPY_TARGET} PRIVATE
${MICROPY_MPVERSION}
${MICROPY_QSTRDEFS_GENERATED}
${MICROPY_MODULEDEFS}
${MICROPY_ROOT_POINTERS}
)
# Command to force the build of another command
@ -139,6 +143,31 @@ add_custom_command(
DEPENDS ${MICROPY_MODULEDEFS_COLLECTED}
)
# Generate root_pointers.h
add_custom_command(
OUTPUT ${MICROPY_ROOT_POINTERS_SPLIT}
COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py split root_pointer ${MICROPY_GENHDR_DIR}/qstr.i.last ${MICROPY_GENHDR_DIR}/root_pointer _
COMMAND touch ${MICROPY_ROOT_POINTERS_SPLIT}
DEPENDS ${MICROPY_QSTRDEFS_LAST}
VERBATIM
COMMAND_EXPAND_LISTS
)
add_custom_command(
OUTPUT ${MICROPY_ROOT_POINTERS_COLLECTED}
COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/makeqstrdefs.py cat root_pointer _ ${MICROPY_GENHDR_DIR}/root_pointer ${MICROPY_ROOT_POINTERS_COLLECTED}
DEPENDS ${MICROPY_ROOT_POINTERS_SPLIT}
VERBATIM
COMMAND_EXPAND_LISTS
)
add_custom_command(
OUTPUT ${MICROPY_ROOT_POINTERS}
COMMAND ${Python3_EXECUTABLE} ${MICROPY_PY_DIR}/make_root_pointers.py ${MICROPY_ROOT_POINTERS_COLLECTED} > ${MICROPY_ROOT_POINTERS}
DEPENDS ${MICROPY_ROOT_POINTERS_COLLECTED} ${MICROPY_PY_DIR}/make_root_pointers.py
)
# Build frozen code if enabled
if(MICROPY_FROZEN_MANIFEST)
@ -174,6 +203,7 @@ if(MICROPY_FROZEN_MANIFEST)
COMMAND ${Python3_EXECUTABLE} ${MICROPY_DIR}/tools/makemanifest.py -o ${MICROPY_FROZEN_CONTENT} -v "MPY_DIR=${MICROPY_DIR}" -v "MPY_LIB_DIR=${MICROPY_LIB_DIR}" -v "PORT_DIR=${MICROPY_PORT_DIR}" -v "BOARD_DIR=${MICROPY_BOARD_DIR}" -b "${CMAKE_BINARY_DIR}" -f${MICROPY_CROSS_FLAGS} ${MICROPY_FROZEN_MANIFEST}
DEPENDS MICROPY_FORCE_BUILD
${MICROPY_QSTRDEFS_GENERATED}
${MICROPY_ROOT_POINTERS}
${MICROPY_MPYCROSS_DEPENDENCY}
VERBATIM
)

16
py/mkrules.mk
View File

@ -7,8 +7,8 @@ endif
# Extra deps that need to happen before object compilation.
OBJ_EXTRA_ORDER_DEPS =
# Generate moduledefs.h.
OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h
# Generate header files.
OBJ_EXTRA_ORDER_DEPS += $(HEADER_BUILD)/moduledefs.h $(HEADER_BUILD)/root_pointers.h
ifeq ($(MICROPY_ROM_TEXT_COMPRESSION),1)
# If compression is enabled, trigger the build of compressed.data.h...
@ -126,6 +126,16 @@ $(HEADER_BUILD)/moduledefs.collected: $(HEADER_BUILD)/moduledefs.split
$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat module _ $(HEADER_BUILD)/module $@
# Module definitions via MP_REGISTER_ROOT_POINTER.
$(HEADER_BUILD)/root_pointers.split: $(HEADER_BUILD)/qstr.i.last
$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py split root_pointer $< $(HEADER_BUILD)/root_pointer _
$(Q)$(TOUCH) $@
$(HEADER_BUILD)/root_pointers.collected: $(HEADER_BUILD)/root_pointers.split
$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py cat root_pointer _ $(HEADER_BUILD)/root_pointer $@
# Compressed error strings.
$(HEADER_BUILD)/compressed.split: $(HEADER_BUILD)/qstr.i.last
$(ECHO) "GEN $@"
@ -165,7 +175,7 @@ endif
ifneq ($(FROZEN_MANIFEST),)
# to build frozen_content.c from a manifest
$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY)
$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h $(BUILD)/genhdr/root_pointers.h | $(MICROPY_MPYCROSS_DEPENDENCY)
$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST)
endif

7
py/mpstate.h
View File

@ -192,6 +192,13 @@ typedef struct _mp_state_vm_t {
// include any root pointers defined by a port
MICROPY_PORT_ROOT_POINTERS
// Include any root pointers registered with MP_REGISTER_ROOT_POINTER().
#ifndef NO_QSTR
// Only include root pointer definitions when not doing qstr extraction, because
// the qstr extraction stage also generates the root pointers header file.
#include "genhdr/root_pointers.h"
#endif
// root pointers for extmod
#if MICROPY_REPL_EVENT_DRIVEN

11
py/obj.h
View File

@ -416,13 +416,18 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t;
#define MP_DEFINE_CONST_STATICMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_staticmethod}, fun_name}
#define MP_DEFINE_CONST_CLASSMETHOD_OBJ(obj_name, fun_name) const mp_rom_obj_static_class_method_t obj_name = {{&mp_type_classmethod}, fun_name}
#ifndef NO_QSTR
// Declare a module as a builtin, processed by makemoduledefs.py
// param module_name: MP_QSTR_<module name>
// param obj_module: mp_obj_module_t instance
#ifndef NO_QSTR
#define MP_REGISTER_MODULE(module_name, obj_module)
#endif
// Declare a root pointer (to avoid garbage collection of a global static variable).
// param variable_declaration: a valid C variable declaration
#define MP_REGISTER_ROOT_POINTER(variable_declaration)
#endif // NO_QSTR
// Underlying map/hash table implementation (not dict object or map function)

5
py/py.mk
View File

@ -218,6 +218,11 @@ $(HEADER_BUILD)/moduledefs.h: $(HEADER_BUILD)/moduledefs.collected
@$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py $< > $@
# build a list of registered root pointers for py/mpstate.h.
$(HEADER_BUILD)/root_pointers.h: $(HEADER_BUILD)/root_pointers.collected $(PY_SRC)/make_root_pointers.py
@$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/make_root_pointers.py $< > $@
# Standard C functions like memset need to be compiled with special flags so
# the compiler does not optimise these functions in terms of themselves.
CFLAGS_BUILTIN ?= -ffreestanding -fno-builtin -fno-lto