circuitpython/py/py.mk

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

323 lines
9.2 KiB
Makefile
Raw Normal View History

# where py object files go (they have a name prefix to prevent filename clashes)
PY_BUILD = $(BUILD)/py
# where autogenerated header files go
HEADER_BUILD = $(BUILD)/genhdr
# file containing qstr defs for the core Python bit
PY_QSTR_DEFS = $(PY_SRC)/qstrdefs.h
TRANSLATION ?= en_US
2018-07-31 19:53:54 -04:00
# If qstr autogeneration is not disabled we specify the output header
# for all collected qstrings.
ifneq ($(QSTR_AUTOGEN_DISABLE),1)
QSTR_DEFS_COLLECTED = $(HEADER_BUILD)/qstrdefs.collected.h
endif
# Any files listed by these variables will cause a full regeneration of qstrs
# DEPENDENCIES: included in qstr processing; REQUIREMENTS: not included
QSTR_GLOBAL_DEPENDENCIES += $(PY_SRC)/mpconfig.h mpconfigport.h
QSTR_GLOBAL_REQUIREMENTS += $(HEADER_BUILD)/mpversion.h
# some code is performance bottleneck and compiled with other optimization options
2020-12-14 18:57:31 -05:00
CSUPEROPT = -O3
# Enable building 32-bit code on 64-bit host.
ifeq ($(MICROPY_FORCE_32BIT),1)
CC += -m32
CXX += -m32
LD += -m32
endif
# External modules written in C.
ifneq ($(USER_C_MODULES),)
# pre-define USERMOD variables as expanded so that variables are immediate
# expanded as they're added to them
SRC_USERMOD :=
SRC_USERMOD_CXX :=
CFLAGS_USERMOD :=
CXXFLAGS_USERMOD :=
LDFLAGS_USERMOD :=
$(foreach module, $(wildcard $(USER_C_MODULES)/*/micropython.mk), \
$(eval USERMOD_DIR = $(patsubst %/,%,$(dir $(module))))\
$(info Including User C Module from $(USERMOD_DIR))\
$(eval include $(module))\
)
SRC_MOD += $(patsubst $(USER_C_MODULES)/%.c,%.c,$(SRC_USERMOD))
SRC_MOD_CXX += $(patsubst $(USER_C_MODULES)/%.cpp,%.cpp,$(SRC_USERMOD_CXX))
CFLAGS_MOD += $(CFLAGS_USERMOD)
CXXFLAGS_MOD += $(CXXFLAGS_USERMOD)
LDFLAGS_MOD += $(LDFLAGS_USERMOD)
endif
ifeq ($(CIRCUITPY_ULAB),1)
ULAB_SRCS := $(shell find $(TOP)/extmod/ulab/code -type f -name "*.c")
SRC_MOD += $(patsubst $(TOP)/%,%,$(ULAB_SRCS))
CFLAGS_MOD += -DCIRCUITPY_ULAB=1 -DMODULE_ULAB_ENABLED=1 -DULAB_HAS_USER_MODULE=0 -iquote $(TOP)/extmod/ulab/code
$(BUILD)/extmod/ulab/code/%.o: CFLAGS += -Wno-missing-declarations -Wno-missing-prototypes -Wno-unused-parameter -Wno-float-equal -Wno-sign-compare -Wno-cast-align -Wno-shadow -DCIRCUITPY
2020-02-04 11:24:37 -05:00
endif
# py object files
PY_CORE_O_BASENAME = $(addprefix py/,\
mpstate.o \
nlr.o \
nlrx86.o \
nlrx64.o \
nlrthumb.o \
2021-02-17 18:28:42 -05:00
nlraarch64.o \
nlrpowerpc.o \
2014-11-27 15:29:33 -05:00
nlrxtensa.o \
nlrsetjmp.o \
malloc.o \
gc.o \
Introduce a long lived section of the heap. This adapts the allocation process to start from either end of the heap when searching for free space. The default behavior is identical to the existing behavior where it starts with the lowest block and looks higher. Now it can also look from the highest block and lower depending on the long_lived parameter to gc_alloc. As the heap fills, the two sections may overlap. When they overlap, a collect may be triggered in order to keep the long lived section compact. However, free space is always eligable for each type of allocation. By starting from either of the end of the heap we have ability to separate short lived objects from long lived ones. This separation reduces heap fragmentation because long lived objects are easy to densely pack. Most objects are short lived initially but may be made long lived when they are referenced by a type or module. This involves copying the memory and then letting the collect phase free the old portion. QSTR pools and chunks are always long lived because they are never freed. The reallocation, collection and free processes are largely unchanged. They simply also maintain an index to the highest free block as well as the lowest. These indices are used to speed up the allocation search until the next collect. In practice, this change may slightly slow down import statements with the benefit that memory is much less fragmented afterwards. For example, a test import into a 20k heap that leaves ~6k free previously had the largest continuous free space of ~400 bytes. After this change, the largest continuous free space is over 3400 bytes.
2018-01-23 19:22:05 -05:00
gc_long_lived.o \
py: Introduce a Python stack for scoped allocation. This patch introduces the MICROPY_ENABLE_PYSTACK option (disabled by default) which enables a "Python stack" that allows to allocate and free memory in a scoped, or Last-In-First-Out (LIFO) way, similar to alloca(). A new memory allocation API is introduced along with this Py-stack. It includes both "local" and "nonlocal" LIFO allocation. Local allocation is intended to be equivalent to using alloca(), whereby the same function must free the memory. Nonlocal allocation is where another function may free the memory, so long as it's still LIFO. Follow-up patches will convert all uses of alloca() and VLA to the new scoped allocation API. The old behaviour (using alloca()) will still be available, but when MICROPY_ENABLE_PYSTACK is enabled then alloca() is no longer required or used. The benefits of enabling this option are (or will be once subsequent patches are made to convert alloca()/VLA): - Toolchains without alloca() can use this feature to obtain correct and efficient scoped memory allocation (compared to using the heap instead of alloca(), which is slower). - Even if alloca() is available, enabling the Py-stack gives slightly more efficient use of stack space when calling nested Python functions, due to the way that compilers implement alloca(). - Enabling the Py-stack with the stackless mode allows for even more efficient stack usage, as well as retaining high performance (because the heap is no longer used to build and destroy stackless code states). - With Py-stack and stackless enabled, Python-calling-Python is no longer recursive in the C mp_execute_bytecode function. The micropython.pystack_use() function is included to measure usage of the Python stack.
2017-11-26 07:28:40 -05:00
pystack.o \
qstr.o \
vstr.o \
mpprint.o \
unicode.o \
mpz.o \
reader.o \
lexer.o \
parse.o \
scope.o \
compile.o \
emitcommon.o \
emitbc.o \
asmbase.o \
asmx64.o \
emitnx64.o \
asmx86.o \
emitnx86.o \
asmthumb.o \
emitnthumb.o \
emitinlinethumb.o \
2014-08-16 16:55:53 -04:00
asmarm.o \
emitnarm.o \
asmxtensa.o \
emitnxtensa.o \
emitinlinextensa.o \
emitnxtensawin.o \
formatfloat.o \
parsenumbase.o \
parsenum.o \
emitglue.o \
persistentcode.o \
runtime.o \
runtime_utils.o \
scheduler.o \
nativeglue.o \
pairheap.o \
ringbuf.o \
stackctrl.o \
argcheck.o \
warning.o \
profile.o \
map.o \
enum.o \
obj.o \
objarray.o \
objattrtuple.o \
objbool.o \
objboundmeth.o \
objcell.o \
objclosure.o \
objcomplex.o \
objdeque.o \
objdict.o \
2014-01-14 18:55:01 -05:00
objenumerate.o \
objexcept.o \
2014-01-14 20:37:08 -05:00
objfilter.o \
objfloat.o \
objfun.o \
objgenerator.o \
objgetitemiter.o \
objint.o \
objint_longlong.o \
objint_mpz.o \
objlist.o \
2014-01-14 20:10:09 -05:00
objmap.o \
objmodule.o \
2014-03-22 17:31:28 -04:00
objobject.o \
objpolyiter.o \
objproperty.o \
objnone.o \
objnamedtuple.o \
objrange.o \
objreversed.o \
objset.o \
objsingleton.o \
objslice.o \
objstr.o \
objstrunicode.o \
2014-04-26 13:26:14 -04:00
objstringio.o \
2021-08-02 02:30:48 -04:00
objtraceback.o \
objtuple.o \
objtype.o \
objzip.o \
opmethods.o \
protocols: Allow them to be (optionally) type-safe Protocols are nice, but there is no way for C code to verify whether a type's "protocol" structure actually implements some particular protocol. As a result, you can pass an object that implements the "vfs" protocol to one that expects the "stream" protocol, and the opposite of awesomeness ensues. This patch adds an OPTIONAL (but enabled by default) protocol identifier as the first member of any protocol structure. This identifier is simply a unique QSTR chosen by the protocol designer and used by each protocol implementer. When checking for protocol support, instead of just checking whether the object's type has a non-NULL protocol field, use `mp_proto_get` which implements the protocol check when possible. The existing protocols are now named: protocol_framebuf protocol_i2c protocol_pin protocol_stream protocol_spi protocol_vfs (most of these are unused in CP and are just inherited from MP; vfs and stream are definitely used though) I did not find any crashing examples, but here's one to give a flavor of what is improved, using `micropython_coverage`. Before the change, the vfs "ioctl" protocol is invoked, and the result is not intelligible as json (but it could have resulted in a hard fault, potentially): >>> import uos, ujson >>> u = uos.VfsPosix('/tmp') >>> ujson.load(u) Traceback (most recent call last): File "<stdin>", line 1, in <module> ValueError: syntax error in JSON After the change, the vfs object is correctly detected as not supporting the stream protocol: >>> ujson.load(p) Traceback (most recent call last): File "<stdin>", line 1, in <module> OSError: stream operation not supported
2019-12-03 15:50:37 -05:00
proto.o \
sequence.o \
stream.o \
binary.o \
builtinimport.o \
2014-02-03 17:44:10 -05:00
builtinevex.o \
builtinhelp.o \
modarray.o \
modbuiltins.o \
modcollections.o \
modgc.o \
modio.o \
modmath.o \
modcmath.o \
modmicropython.o \
modstruct.o \
modsys.o \
moduerrno.o \
modthread.o \
vm.o \
bc.o \
showbc.o \
repl.o \
smallint.o \
frozenmod.o \
)
PY_EXTMOD_O_BASENAME = \
extmod/moduasyncio.o \
extmod/moductypes.o \
extmod/modujson.o \
extmod/modure.o \
extmod/moduzlib.o \
extmod/moduheapq.o \
extmod/modutimeq.o \
extmod/moduhashlib.o \
extmod/modubinascii.o \
extmod/modurandom.o \
extmod/moduselect.o \
extmod/modframebuf.o \
extmod/vfs.o \
extmod/vfs_blockdev.o \
extmod/vfs_reader.o \
extmod/vfs_posix.o \
extmod/vfs_posix_file.o \
extmod/vfs_fat.o \
extmod/vfs_fat_diskio.o \
extmod/vfs_fat_file.o \
extmod/vfs_lfs.o \
extmod/utime_mphal.o \
shared/libc/abort_.o \
shared/libc/printf.o \
# prepend the build destination prefix to the py object files
PY_CORE_O = $(addprefix $(BUILD)/, $(PY_CORE_O_BASENAME))
PY_EXTMOD_O = $(addprefix $(BUILD)/, $(PY_EXTMOD_O_BASENAME))
# this is a convenience variable for ports that want core, extmod and frozen code
PY_O = $(PY_CORE_O) $(PY_EXTMOD_O)
# object file for frozen code specified via a manifest
ifneq ($(FROZEN_MANIFEST),)
PY_O += $(BUILD)/$(BUILD)/frozen_content.o
endif
# Sources that may contain qstrings
SRC_QSTR_IGNORE = py/nlr%
SRC_QSTR_EMITNATIVE = py/emitn%
SRC_QSTR += $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) $(PY_EXTMOD_O_BASENAME:.o=.c)
# Sources that only hold QSTRs after pre-processing.
SRC_QSTR_PREPROCESSOR = $(addprefix $(TOP)/, $(filter $(SRC_QSTR_EMITNATIVE),$(PY_CORE_O_BASENAME:.o=.c)))
2014-04-03 19:32:58 -04:00
# Anything that depends on FORCE will be considered out-of-date
FORCE:
.PHONY: FORCE
$(HEADER_BUILD)/mpversion.h: FORCE | $(HEADER_BUILD)
2018-07-31 19:53:54 -04:00
$(STEPECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeversionhdr.py $@
2019-04-05 15:39:44 -04:00
# mpconfigport.mk is optional, but changes to it may drastically change
# overall config, so they need to be caught
MPCONFIGPORT_MK = $(wildcard mpconfigport.mk)
$(HEADER_BUILD)/$(TRANSLATION).mo: $(TOP)/locale/$(TRANSLATION).po | $(HEADER_BUILD)
2023-02-13 00:12:24 -05:00
$(Q)$(PYTHON) $(TOP)/tools/msgfmt.py -o $@ $^
2018-07-31 19:53:54 -04:00
$(HEADER_BUILD)/qstrdefs.preprocessed.h: $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) mpconfigport.h $(MPCONFIGPORT_MK) $(PY_SRC)/mpconfig.h | $(HEADER_BUILD)
$(STEPECHO) "GEN $@"
$(Q)cat $(PY_QSTR_DEFS) $(QSTR_DEFS) $(QSTR_DEFS_COLLECTED) | $(SED) 's/^Q(.*)/"&"/' | $(CPP) $(CFLAGS) - | $(SED) 's/^"\(Q(.*)\)"/\1/' > $@
# qstr data
2018-07-31 19:53:54 -04:00
$(HEADER_BUILD)/qstrdefs.enum.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h
$(STEPECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py --output_type=enums $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
2018-07-31 19:53:54 -04:00
# Adding an order only dependency on $(HEADER_BUILD) causes $(HEADER_BUILD) to get
# created before we run the script to generate the .h
# Note: we need to protect the qstr names from the preprocessor, so we wrap
# the lines in "" and then unwrap after the preprocessor is finished.
$(HEADER_BUILD)/qstrdefs.generated.h: $(PY_SRC)/makeqstrdata.py $(HEADER_BUILD)/qstrdefs.preprocessed.h
2018-07-31 19:53:54 -04:00
$(STEPECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makeqstrdata.py --output_type=data $(HEADER_BUILD)/qstrdefs.preprocessed.h > $@
2018-07-31 19:53:54 -04:00
2022-11-08 18:05:09 -05:00
# Is generated as a side-effect of building compression.generated.h
# Specifying both in a single rule actually causes the rule to be run twice!
# This alternative makes it run just once.
$(PY_BUILD)/translations-$(TRANSLATION).c: $(HEADER_BUILD)/compression.generated.h
@true
$(HEADER_BUILD)/compression.generated.h: $(PY_SRC)/maketranslationdata.py $(HEADER_BUILD)/$(TRANSLATION).mo $(HEADER_BUILD)/qstrdefs.preprocessed.h
$(STEPECHO) "GEN $@"
2022-11-08 18:05:09 -05:00
$(Q)mkdir -p $(PY_BUILD)
$(Q)$(PYTHON) $(PY_SRC)/maketranslationdata.py --compression_filename $(HEADER_BUILD)/compression.generated.h --translation $(HEADER_BUILD)/$(TRANSLATION).mo --translation_filename $(PY_BUILD)/translations-$(TRANSLATION).c $(HEADER_BUILD)/qstrdefs.preprocessed.h
2022-06-03 18:06:58 -04:00
PY_CORE_O += $(PY_BUILD)/translations-$(TRANSLATION).o
$(PY_BUILD)/qstr.o: $(HEADER_BUILD)/qstrdefs.generated.h
py: Implement "common word" compression scheme for error messages. The idea here is that there's a moderate amount of ROM used up by exception text. Obviously we try to keep the messages short, and the code can enable terse errors, but it still adds up. Listed below is the total string data size for various ports: bare-arm 2860 minimal 2876 stm32 8926 (PYBV11) cc3200 3751 esp32 5721 This commit implements compression of these strings. It takes advantage of the fact that these strings are all 7-bit ascii and extracts the top 128 frequently used words from the messages and stores them packed (dropping their null-terminator), then uses (0x80 | index) inside strings to refer to these common words. Spaces are automatically added around words, saving more bytes. This happens transparently in the build process, mirroring the steps that are used to generate the QSTR data. The MP_COMPRESSED_ROM_TEXT macro wraps any literal string that should compressed, and it's automatically decompressed in mp_decompress_rom_string. There are many schemes that could be used for the compression, and some are included in py/makecompresseddata.py for reference (space, Huffman, ngram, common word). Results showed that the common-word compression gets better results. This is before counting the increased cost of the Huffman decoder. This might be slightly counter-intuitive, but this data is extremely repetitive at a word-level, and the byte-level entropy coder can't quite exploit that as efficiently. Ideally one would combine both approaches, but for now the common-word approach is the one that is used. For additional comparison, the size of the raw data compressed with gzip and zlib is calculated, as a sort of proxy for a lower entropy bound. With this scheme we come within 15% on stm32, and 30% on bare-arm (i.e. we use x% more bytes than the data compressed with gzip -- not counting the code overhead of a decoder, and how this would be hypothetically implemented). The feature is disabled by default and can be enabled by setting MICROPY_ROM_TEXT_COMPRESSION at the Makefile-level.
2019-09-26 08:19:29 -04:00
# build a list of registered modules for py/objmodule.c.
$(HEADER_BUILD)/moduledefs.h: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(HEADER_BUILD)/mpversion.h
@$(ECHO) "GEN $@"
$(Q)$(PYTHON) $(PY_SRC)/makemoduledefs.py --vpath="., $(TOP), $(USER_C_MODULES)" $(SRC_QSTR) > $@
# Standard C functions like memset need to be compiled with special flags so
# the compiler does not optimise these functions in terms of themselves.
CFLAGS_BUILTIN ?= -ffreestanding -fno-builtin -fno-lto
$(BUILD)/shared/libc/string0.o: CFLAGS += $(CFLAGS_BUILTIN)
# Force nlr code to always be compiled with space-saving optimisation so
# that the function preludes are of a minimal and predictable form.
$(PY_BUILD)/nlr%.o: CFLAGS += -Os
# optimising gc for speed; 5ms down to 4ms on pybv2
ifndef SUPEROPT_GC
SUPEROPT_GC = 1
endif
ifeq ($(SUPEROPT_GC),1)
$(PY_BUILD)/gc.o: CFLAGS += $(CSUPEROPT)
endif
# optimising vm for speed, adds only a small amount to code size but makes a huge difference to speed (20% faster)
ifndef SUPEROPT_VM
SUPEROPT_VM = 1
endif
ifeq ($(SUPEROPT_VM),1)
$(PY_BUILD)/vm.o: CFLAGS += $(CSUPEROPT)
endif
# Optimizing vm.o for modern deeply pipelined CPUs with branch predictors
# may require disabling tail jump optimization. This will make sure that
# each opcode has its own dispatching jump which will improve branch
# branch predictor efficiency.
2018-11-30 16:23:53 -05:00
# https://marc.info/?l=lua-l&m=129778596120851
# http://hg.python.org/cpython/file/b127046831e2/Python/ceval.c#l828
# http://www.emulators.com/docs/nx25_nostradamus.htm
#-fno-crossjumping
# Include rules for extmod related code
include $(TOP)/extmod/extmod.mk