Added ability to freeze multiple directories; freeze neopixel library in cpx build (#199)

Reworked frozen module support: clean up makefiles and handle multiple directories.
Modules to freeze are included as git submodules.

Add neopixel to circuitplayground express build.

Fixes #56

.gitmodules

@@ -17,3 +17,6 @@
 [submodule "tools/uf2"]
 	path = tools/uf2
 	url = https://github.com/Microsoft/uf2.git
+[submodule "atmel-samd/frozen/Adafruit_CircuitPython_NeoPixel"]
+	path = frozen/Adafruit_CircuitPython_NeoPixel
+	url = https://github.com/adafruit/Adafruit_CircuitPython_NeoPixel

atmel-samd/Makefile

@@ -145,9 +145,11 @@ CFLAGS += -DMICROPY_MODULE_FROZEN_STR
 CFLAGS += -Wno-error=lto-type-mismatch
 endif
 
-ifneq ($(FROZEN_MPY_DIR),)
 # To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and
-# then invoke make with FROZEN_MPY_DIR=frozen (be sure to build from scratch).
+# then invoke make with FROZEN_MPY_DIR=frozen or FROZEN_MPY_DIRS="dir1 dir2"
+# (be sure to build from scratch).
+
+ifneq ($(FROZEN_MPY_DIRS),)
 CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool
 CFLAGS += -DMICROPY_MODULE_FROZEN_MPY
 CFLAGS += -Wno-error=lto-type-mismatch

atmel-samd/boards/circuitplayground_express/mpconfigboard.mk

@@ -5,3 +5,6 @@ USB_PID = 0x8019
 FLASH_IMPL = spi_flash.c
 
 CHIP_VARIANT = SAMD21G18A
+
+# Include these Python libraries in firmware.
+FROZEN_MPY_DIRS += $(TOP)/frozen/Adafruit_CircuitPython_NeoPixel

frozen/Adafruit_CircuitPython_NeoPixel

@@ -0,0 +1 @@
+Subproject commit 1142f1c7fdc232a46d74dd4f1946a5f462ae2555

py/mkenv.mk

@@ -42,12 +42,16 @@ endif
 PY_SRC ?= $(TOP)/py
 BUILD ?= build
 
-RM = rm
 ECHO = @echo
+
+CD = cd
 CP = cp
+FIND = find
 MKDIR = mkdir
-SED = sed
 PYTHON = python
+RM = rm
+RSYNC = rsync
+SED = sed
 
 AS = $(CROSS_COMPILE)as
 CC = $(CROSS_COMPILE)gcc

py/mkrules.mk

@@ -101,29 +101,30 @@ $(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DE
 	$(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@
 endif
 
-ifneq ($(FROZEN_MPY_DIR),)
+ifneq ($(FROZEN_MPY_DIRS),)
 # to build the MicroPython cross compiler
-$(TOP)/mpy-cross/mpy-cross: $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c
+# Currently not used, because the wrong mpy-cross may be left over from a previous build. Build by hand to make sure.
+$(MPY_CROSS): $(TOP)/py/*.[ch] $(TOP)/mpy-cross/*.[ch] $(TOP)/windows/fmode.c
 	$(Q)$(MAKE) -C $(TOP)/mpy-cross
 
-# make a list of all the .py files that need compiling and freezing
+# Copy all the modules and single python files to freeze to a common area, omitting top-level dirs (the repo names).
-BLAH := $(info $(shell pwd))
+# Remove any conf.py (sphinx config) and setup.py (module install info) files, which are not meant to be frozen.
-
+# Then compile .mpy files from all the .py files, placing them in the same directories as the .py files.
-FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==')
+$(BUILD)/frozen_mpy: $(FROZEN_MPY_DIRS)
-FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/frozen_mpy/,$(FROZEN_MPY_PY_FILES:.py=.mpy))
+	$(ECHO) FREEZE $(FROZEN_MPY_DIRS)
-
+	$(Q)$(MKDIR) -p $@
-# to build .mpy files from .py files
+	$(Q)$(RSYNC) -rL --include="*/" --include='*.py' --exclude="*" $(addsuffix /*,$(FROZEN_MPY_DIRS)) $@
-$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py
+	$(Q)$(RM) -f $@/conf.py $@/setup.py
-	@$(ECHO) "MPY $<"
+	$(Q)$(CD) $@ && \
-	$(Q)$(MKDIR) -p $(dir $@)
+$(FIND) -L . -type f -name '*.py' | sed 's=^\./==' | \
-	$(Q)$(MPY_CROSS) -o $@ -s $(<:$(FROZEN_MPY_DIR)/%=%) $(MPY_CROSS_FLAGS) $<
+xargs -n1 $(abspath $(MPY_CROSS)) $(MPY_CROSS_FLAGS)
 
 # to build frozen_mpy.c from all .mpy files
 # You need to define MPY_TOOL_LONGINT_IMPL in mpconfigport.mk
 # if the default will not work (mpz is the default).
-$(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generated.h
+$(BUILD)/frozen_mpy.c: $(BUILD)/frozen_mpy $(BUILD)/genhdr/qstrdefs.generated.h
 	$(STEPECHO) "Creating $@"
-	$(Q)$(PYTHON) $(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@
+	$(Q)$(PYTHON) $(MPY_TOOL) $(MPY_TOOL_LONGINT_IMPL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(shell $(FIND) -L $(BUILD)/frozen_mpy -type f -name '*.mpy') > $@
 endif
 
 ifneq ($(PROG),)

py/py.mk

@@ -247,12 +247,15 @@ PY_O = $(addprefix $(PY_BUILD)/, $(PY_O_BASENAME))
 
 # object file for frozen files
 ifneq ($(FROZEN_DIR),)
-PY_O += $(BUILD)/$(BUILD)/frozen.o
+PY_O += $(BUILD)/frozen.o
 endif
 
+# Combine old singular FROZEN_MPY_DIR with new multiple value form.
+FROZEN_MPY_DIRS += $(FROZEN_MPY_DIR)
+
 # object file for frozen bytecode (frozen .mpy files)
-ifneq ($(FROZEN_MPY_DIR),)
+ifneq ($(FROZEN_MPY_DIRS),)
-PY_O += $(BUILD)/$(BUILD)/frozen_mpy.o
+PY_O += $(BUILD)/frozen_mpy.o
 endif
 
 # Sources that may contain qstrings